[llvm] [CostModel][X86] Update logic vector reduction costs to match llvm-mca estimates (PR #214191)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 03:38:52 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/214191
>From e7f0271ff958398bd994db0cec485b99897c7661 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 5 Aug 2026 11:35:50 +0100
Subject: [PATCH 1/2] [CostModel][X86] Update logic vector reduction costs to
match llvm-mca estimates
Replaces the olds costs which were just instruction counts
Still working on the bool reduction costs - these will be handled in a followup
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 88 ++++-
.../test/Analysis/CostModel/X86/reduce-and.ll | 186 ++++++-----
llvm/test/Analysis/CostModel/X86/reduce-or.ll | 186 ++++++-----
.../test/Analysis/CostModel/X86/reduce-xor.ll | 186 ++++++-----
llvm/test/Transforms/PhaseOrdering/X86/avg.ll | 156 ++-------
.../Transforms/SLPVectorizer/X86/bool-mask.ll | 38 +--
.../X86/extractelemets-extended-by-poison.ll | 27 +-
.../X86/multi-use-bitcasted-reduction.ll | 28 +-
...duced-val-extracted-and-externally-used.ll | 22 +-
.../X86/replaced-external-in-reduction.ll | 20 +-
.../X86/same-values-sub-node-with-poisons.ll | 34 +-
.../X86/fold-equivalent-reduction-cmp.ll | 18 +-
.../X86/fold-reduction-zero-test.ll | 126 ++++----
.../fold-signbit-reduction-cmp-codesize.ll | 2 +-
.../X86/fold-signbit-reduction-cmp.ll | 301 +++++++++---------
.../VectorCombine/X86/icmp-vector-reduce.ll | 48 ++-
16 files changed, 732 insertions(+), 734 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index a421a02fd15db..4f424517e8d08 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5696,8 +5696,9 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
if (TTI::requiresOrderedReduction(FMF))
return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
+ // We use llvm-mca across all supported CPUs to measure the logic cost stats.
// We use the Intel Architecture Code Analyzer(IACA) to measure the throughput
- // and make it as the cost.
+ // and make it as the cost. TODO: Update old IACA numbers to llvm-mca.
static const CostKindTblEntry SLMCostTbl[] = {
{ ISD::FADD, MVT::v2f64, {3, 3, 3, 3} },
@@ -5718,6 +5719,19 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::ADD, MVT::v4i8, {2, 2, 2, 2} },
{ ISD::ADD, MVT::v8i8, {2, 2, 2, 2} },
{ ISD::ADD, MVT::v16i8, {3, 3, 3, 3} },
+
+ { ISD::AND, MVT::v2i64, {2, 2, 3, 3} },
+ { ISD::AND, MVT::v4i32, {3, 4, 5, 5} },
+ { ISD::AND, MVT::v8i16, {4, 7, 8, 8} },
+ { ISD::AND, MVT::v16i8, {5,10,11,11} },
+ { ISD::OR, MVT::v2i64, {2, 2, 3, 3} },
+ { ISD::OR, MVT::v4i32, {3, 4, 5, 5} },
+ { ISD::OR, MVT::v8i16, {4, 7, 8, 8} },
+ { ISD::OR, MVT::v16i8, {5,10,11,11} },
+ { ISD::XOR, MVT::v2i64, {2, 2, 3, 3} },
+ { ISD::XOR, MVT::v4i32, {3, 4, 5, 5} },
+ { ISD::XOR, MVT::v8i16, {4, 7, 8, 8} },
+ { ISD::XOR, MVT::v16i8, {5,10,11,11} },
};
static const CostKindTblEntry AVX1CostTbl[] = {
@@ -5729,6 +5743,52 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::ADD, MVT::v8i32, {5, 5, 5, 5} },
{ ISD::ADD, MVT::v16i16, {5, 5, 5, 5} },
{ ISD::ADD, MVT::v32i8, {4, 4, 4, 4} },
+
+ { ISD::AND, MVT::v4i64, {3, 7, 5, 5} },
+ { ISD::AND, MVT::v8i32, {4, 9, 7, 7} },
+ { ISD::AND, MVT::v16i16, {5,11, 9, 9} },
+ { ISD::AND, MVT::v8i16, {4, 7, 7, 7} },
+ { ISD::AND, MVT::v32i8, {6,13,11,11} },
+ { ISD::AND, MVT::v16i8, {5,10, 9, 9} },
+ { ISD::OR, MVT::v4i64, {3, 7, 5, 5} },
+ { ISD::OR, MVT::v8i32, {4, 9, 7, 7} },
+ { ISD::OR, MVT::v16i16, {5,11, 9, 9} },
+ { ISD::OR, MVT::v8i16, {4, 7, 7, 7} },
+ { ISD::OR, MVT::v32i8, {6,13,11,11} },
+ { ISD::OR, MVT::v16i8, {5,10, 9, 9} },
+ { ISD::XOR, MVT::v4i64, {3, 7, 5, 5} },
+ { ISD::XOR, MVT::v8i32, {4, 9, 7, 7} },
+ { ISD::XOR, MVT::v16i16, {5,11, 9, 9} },
+ { ISD::XOR, MVT::v8i16, {4, 7, 7, 7} },
+ { ISD::XOR, MVT::v32i8, {6,13,11,11} },
+ { ISD::XOR, MVT::v16i8, {5,10, 9, 9} },
+ };
+
+ static const CostKindTblEntry AVX2CostTbl[] = {
+ { ISD::AND, MVT::v4i64, {2, 7, 5, 5} },
+ { ISD::AND, MVT::v2i64, {1, 2, 3, 3} },
+ { ISD::AND, MVT::v8i32, {3, 9, 7, 7} },
+ { ISD::AND, MVT::v4i32, {2, 4, 5, 5} },
+ { ISD::AND, MVT::v16i16, {3,11, 9, 9} },
+ { ISD::AND, MVT::v8i16, {2, 6, 7, 7} },
+ { ISD::AND, MVT::v32i8, {3,13,11,11} },
+ { ISD::AND, MVT::v16i8, {3, 8, 9, 9} },
+ { ISD::OR, MVT::v4i64, {2, 7, 5, 5} },
+ { ISD::OR, MVT::v2i64, {1, 2, 3, 3} },
+ { ISD::OR, MVT::v8i32, {3, 9, 7, 7} },
+ { ISD::OR, MVT::v4i32, {2, 4, 5, 5} },
+ { ISD::OR, MVT::v16i16, {3,11, 9, 9} },
+ { ISD::OR, MVT::v8i16, {2, 6, 7, 7} },
+ { ISD::OR, MVT::v32i8, {3,13,11,11} },
+ { ISD::OR, MVT::v16i8, {3, 8, 9, 9} },
+ { ISD::XOR, MVT::v4i64, {2, 7, 5, 5} },
+ { ISD::XOR, MVT::v2i64, {1, 2, 3, 3} },
+ { ISD::XOR, MVT::v8i32, {3, 9, 7, 7} },
+ { ISD::XOR, MVT::v4i32, {2, 4, 5, 5} },
+ { ISD::XOR, MVT::v16i16, {3,11, 9, 9} },
+ { ISD::XOR, MVT::v8i16, {2, 6, 7, 7} },
+ { ISD::XOR, MVT::v32i8, {3,13,11,11} },
+ { ISD::XOR, MVT::v16i8, {3, 8, 9, 9} },
};
static const CostKindTblEntry AVX512FCostTbl[] = {
@@ -5736,6 +5796,22 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::FADD, MVT::v16f32, {5, 5, 5, 5} },
{ ISD::ADD, MVT::v8i64, {4, 4, 4, 4} },
{ ISD::ADD, MVT::v16i32, {6, 6, 6, 6} },
+
+ { ISD::AND, MVT::v8i64, {3,10, 7, 7} },
+ { ISD::AND, MVT::v16i32, {4,12, 9, 9} },
+ { ISD::AND, MVT::v32i16, {4,14,11,11} },
+ { ISD::AND, MVT::v64i8, {4,16,13,13} },
+ { ISD::AND, MVT::v16i8, {2, 8, 9, 9} },
+ { ISD::OR, MVT::v8i64, {3,10, 7, 7} },
+ { ISD::OR, MVT::v16i32, {4,12, 9, 9} },
+ { ISD::OR, MVT::v32i16, {4,14,11,11} },
+ { ISD::OR, MVT::v64i8, {4,16,13,13} },
+ { ISD::OR, MVT::v16i8, {2, 8, 9, 9} },
+ { ISD::XOR, MVT::v8i64, {3,10, 7, 7} },
+ { ISD::XOR, MVT::v16i32, {4,12, 9, 9} },
+ { ISD::XOR, MVT::v32i16, {4,14,11,11} },
+ { ISD::XOR, MVT::v64i8, {4,16,13,13} },
+ { ISD::XOR, MVT::v16i8, {2, 8, 9, 9} },
};
static const CostKindTblEntry AVX512BWCostTbl[] = {
@@ -5767,6 +5843,11 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
if (auto KindCost = Entry->Cost[CostKind])
return *KindCost;
+ if (ST->hasAVX2())
+ if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return *KindCost;
+
if (ST->hasAVX())
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
@@ -5898,6 +5979,11 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
if (auto KindCost = Entry->Cost[CostKind])
return ArithmeticCost + *KindCost;
+ if (ST->hasAVX2())
+ if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return ArithmeticCost + *KindCost;
+
if (ST->hasAVX())
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-and.ll b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
index 21338a12381fc..72e10f07c0039 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
@@ -11,26 +11,34 @@
define i32 @reduce_i64(i32 %arg) {
; SSE-LABEL: 'reduce_i64'
; SSE-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 4 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:4 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:10 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i64'
-; AVX-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:6 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i64'
+; AVX1-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i64'
+; AVX2-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i64'
; AVX512-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:10 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:11 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.and.v16i64(<16 x i64> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.and.v1i64(<1 x i64> undef)
@@ -43,27 +51,35 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; SSE-LABEL: 'reduce_i32'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:7 SizeLat:8 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:10 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i32'
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 7 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i32'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i32'
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:9 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:9 Lat:12 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:10 Lat:13 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.and.v32i32(<32 x i32> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
@@ -76,39 +92,39 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; SSE-LABEL: 'reduce_i16'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 14 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:11 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:15 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:6 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:8 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:11 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:13 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i16'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:16 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:11 Lat:14 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.and.v32i16(<32 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:12 Lat:15 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.and.v64i16(<64 x i16> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> undef)
@@ -122,43 +138,43 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 16 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:14 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i8'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:13 CodeSize:13 Lat:17 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:18 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:13 Lat:16 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:14 Lat:17 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
@@ -284,3 +300,5 @@ declare i1 @llvm.vector.reduce.and.v16i1(<16 x i1>)
declare i1 @llvm.vector.reduce.and.v32i1(<32 x i1>)
declare i1 @llvm.vector.reduce.and.v64i1(<64 x i1>)
declare i1 @llvm.vector.reduce.and.v128i1(<128 x i1>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-or.ll b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
index 4b82bb9c685e5..86558a7fe85ba 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
@@ -11,26 +11,34 @@
define i32 @reduce_i64(i32 %arg) {
; SSE-LABEL: 'reduce_i64'
; SSE-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 4 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:4 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:10 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i64'
-; AVX-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:6 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i64'
+; AVX1-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i64'
+; AVX2-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i64'
; AVX512-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:10 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:11 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.or.v1i64(<1 x i64> undef)
@@ -43,27 +51,35 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; SSE-LABEL: 'reduce_i32'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:7 SizeLat:8 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:10 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i32'
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 7 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i32'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i32'
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:9 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:9 Lat:12 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:10 Lat:13 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.or.v32i32(<32 x i32> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
@@ -76,39 +92,39 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; SSE-LABEL: 'reduce_i16'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 14 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:11 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:15 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:6 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:8 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:11 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:13 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i16'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:16 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:11 Lat:14 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.or.v32i16(<32 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:12 Lat:15 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.or.v64i16(<64 x i16> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> undef)
@@ -122,43 +138,43 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 16 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:14 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i8'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:13 CodeSize:13 Lat:17 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:18 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:13 Lat:16 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:14 Lat:17 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
@@ -284,3 +300,5 @@ declare i1 @llvm.vector.reduce.or.v16i1(<16 x i1>)
declare i1 @llvm.vector.reduce.or.v32i1(<32 x i1>)
declare i1 @llvm.vector.reduce.or.v64i1(<64 x i1>)
declare i1 @llvm.vector.reduce.or.v128i1(<128 x i1>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
index c9cb6ef547e86..7f6187cdf7092 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
@@ -11,26 +11,34 @@
define i32 @reduce_i64(i32 %arg) {
; SSE-LABEL: 'reduce_i64'
; SSE-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 4 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:4 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:10 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i64'
-; AVX-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:6 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i64'
+; AVX1-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i64'
+; AVX2-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:6 Lat:8 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:11 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i64'
; AVX512-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:2 SizeLat:3 for: %V2 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:10 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:11 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.xor.v16i64(<16 x i64> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.xor.v1i64(<1 x i64> undef)
@@ -43,27 +51,35 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; SSE-LABEL: 'reduce_i32'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 6 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:5 SizeLat:6 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:8 Lat:7 SizeLat:8 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:10 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'reduce_i32'
-; AVX-NEXT: Cost Model: Found costs of 3 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 7 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:8 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'reduce_i32'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'reduce_i32'
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:13 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i32'
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:9 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:4 SizeLat:5 for: %V4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:9 Lat:12 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:10 Lat:13 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.xor.v32i32(<32 x i32> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
@@ -76,39 +92,39 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; SSE-LABEL: 'reduce_i16'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 8 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
-; SSE-NEXT: Cost Model: Found costs of 14 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:8 Lat:7 SizeLat:8 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:11 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:15 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i16'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:6 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:8 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:11 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:13 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:7 Lat:7 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i16'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:10 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:10 Lat:12 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:14 SizeLat:15 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i16'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:11 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:16 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V4 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:7 Lat:6 SizeLat:7 for: %V8 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:11 Lat:14 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.xor.v32i16(<32 x i16> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:12 Lat:15 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.xor.v64i16(<64 x i16> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> undef)
@@ -122,43 +138,43 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 10 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 12 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of 16 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
-; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:7 SizeLat:5 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:11 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; AVX1-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:5 CodeSize:9 Lat:10 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'reduce_i8'
-; AVX2-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 7 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of 11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:12 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:14 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:12 Lat:14 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:14 Lat:16 SizeLat:17 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'reduce_i8'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of 5 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:13 CodeSize:13 Lat:17 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:18 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:2 CodeSize:9 Lat:8 SizeLat:9 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:11 Lat:13 SizeLat:11 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:4 CodeSize:13 Lat:16 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:14 Lat:17 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
@@ -306,3 +322,5 @@ declare i1 @llvm.vector.reduce.xor.v16i1(<16 x i1>)
declare i1 @llvm.vector.reduce.xor.v32i1(<32 x i1>)
declare i1 @llvm.vector.reduce.xor.v64i1(<64 x i1>)
declare i1 @llvm.vector.reduce.xor.v128i1(<128 x i1>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 8163ae865d2e8..e7d927156a9f1 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -539,129 +539,41 @@ define { i64, i64 } @avgr_16_u8_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
;
; SSE4-LABEL: @avgr_16_u8_alt(
; SSE4-NEXT: entry:
-; SSE4-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
-; SSE4-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[A_COERCE1:%.*]], i64 1
-; SSE4-NEXT: [[TMP2:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i8>
-; SSE4-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 8)
-; SSE4-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 16)
-; SSE4-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 24)
-; SSE4-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 32)
-; SSE4-NEXT: [[TMP7:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 40)
-; SSE4-NEXT: [[A_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE4-NEXT: [[A_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_2_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
-; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_8_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_COERCE0:%.*]] = lshr i64 [[A_COERCE1]], 48
-; SSE4-NEXT: [[B_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_COERCE0]] to i8
-; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
-; SSE4-NEXT: [[A_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_17_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE1:%.*]], i64 0
-; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[B_COERCE2:%.*]], i64 1
-; SSE4-NEXT: [[TMP10:%.*]] = trunc <2 x i64> [[TMP9]] to <2 x i8>
-; SSE4-NEXT: [[TMP11:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 8)
-; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 16)
-; SSE4-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 24)
-; SSE4-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 32)
-; SSE4-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 40)
-; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
-; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_16_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_17_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_11_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2]], 48
-; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_11_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT1:%.*]] = lshr i64 [[B_COERCE2]], 56
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_TRUNC1:%.*]] = trunc nuw i64 [[B_SROA_17_8_EXTRACT_SHIFT1]] to i8
-; SSE4-NEXT: [[TMP16:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i8>
-; SSE4-NEXT: [[TMP17:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i8>
-; SSE4-NEXT: [[TMP18:%.*]] = trunc <2 x i64> [[TMP5]] to <2 x i8>
-; SSE4-NEXT: [[TMP19:%.*]] = trunc <2 x i64> [[TMP6]] to <2 x i8>
-; SSE4-NEXT: [[TMP20:%.*]] = trunc <2 x i64> [[TMP7]] to <2 x i8>
-; SSE4-NEXT: [[TMP21:%.*]] = trunc <2 x i64> [[TMP11]] to <2 x i8>
-; SSE4-NEXT: [[TMP22:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i8>
-; SSE4-NEXT: [[TMP23:%.*]] = trunc <2 x i64> [[TMP13]] to <2 x i8>
-; SSE4-NEXT: [[TMP24:%.*]] = trunc <2 x i64> [[TMP14]] to <2 x i8>
-; SSE4-NEXT: [[TMP25:%.*]] = trunc <2 x i64> [[TMP15]] to <2 x i8>
-; SSE4-NEXT: [[TMP26:%.*]] = lshr <2 x i8> [[TMP2]], splat (i8 1)
-; SSE4-NEXT: [[TMP27:%.*]] = lshr <2 x i8> [[TMP10]], splat (i8 1)
-; SSE4-NEXT: [[TMP28:%.*]] = add nuw <2 x i8> [[TMP27]], [[TMP26]]
-; SSE4-NEXT: [[TMP29:%.*]] = or <2 x i8> [[TMP10]], [[TMP2]]
-; SSE4-NEXT: [[TMP30:%.*]] = and <2 x i8> [[TMP29]], splat (i8 1)
-; SSE4-NEXT: [[TMP31:%.*]] = add nuw <2 x i8> [[TMP28]], [[TMP30]]
-; SSE4-NEXT: [[TMP32:%.*]] = lshr <2 x i8> [[TMP16]], splat (i8 1)
-; SSE4-NEXT: [[TMP33:%.*]] = lshr <2 x i8> [[TMP21]], splat (i8 1)
-; SSE4-NEXT: [[TMP34:%.*]] = add nuw <2 x i8> [[TMP33]], [[TMP32]]
-; SSE4-NEXT: [[TMP35:%.*]] = or <2 x i8> [[TMP21]], [[TMP16]]
-; SSE4-NEXT: [[TMP36:%.*]] = and <2 x i8> [[TMP35]], splat (i8 1)
-; SSE4-NEXT: [[TMP37:%.*]] = add nuw <2 x i8> [[TMP34]], [[TMP36]]
-; SSE4-NEXT: [[TMP38:%.*]] = lshr <2 x i8> [[TMP17]], splat (i8 1)
-; SSE4-NEXT: [[TMP39:%.*]] = lshr <2 x i8> [[TMP22]], splat (i8 1)
-; SSE4-NEXT: [[TMP40:%.*]] = add nuw <2 x i8> [[TMP39]], [[TMP38]]
-; SSE4-NEXT: [[TMP41:%.*]] = or <2 x i8> [[TMP22]], [[TMP17]]
-; SSE4-NEXT: [[TMP42:%.*]] = and <2 x i8> [[TMP41]], splat (i8 1)
-; SSE4-NEXT: [[TMP43:%.*]] = add nuw <2 x i8> [[TMP40]], [[TMP42]]
-; SSE4-NEXT: [[TMP44:%.*]] = lshr <2 x i8> [[TMP18]], splat (i8 1)
-; SSE4-NEXT: [[TMP45:%.*]] = lshr <2 x i8> [[TMP23]], splat (i8 1)
-; SSE4-NEXT: [[TMP46:%.*]] = add nuw <2 x i8> [[TMP45]], [[TMP44]]
-; SSE4-NEXT: [[TMP47:%.*]] = or <2 x i8> [[TMP23]], [[TMP18]]
-; SSE4-NEXT: [[TMP48:%.*]] = and <2 x i8> [[TMP47]], splat (i8 1)
-; SSE4-NEXT: [[TMP104:%.*]] = add nuw <2 x i8> [[TMP46]], [[TMP48]]
-; SSE4-NEXT: [[TMP50:%.*]] = lshr <2 x i8> [[TMP19]], splat (i8 1)
-; SSE4-NEXT: [[TMP51:%.*]] = lshr <2 x i8> [[TMP24]], splat (i8 1)
-; SSE4-NEXT: [[TMP52:%.*]] = add nuw <2 x i8> [[TMP51]], [[TMP50]]
-; SSE4-NEXT: [[TMP53:%.*]] = or <2 x i8> [[TMP24]], [[TMP19]]
-; SSE4-NEXT: [[TMP54:%.*]] = and <2 x i8> [[TMP53]], splat (i8 1)
-; SSE4-NEXT: [[TMP55:%.*]] = add nuw <2 x i8> [[TMP52]], [[TMP54]]
-; SSE4-NEXT: [[TMP56:%.*]] = lshr <2 x i8> [[TMP20]], splat (i8 1)
-; SSE4-NEXT: [[TMP57:%.*]] = lshr <2 x i8> [[TMP25]], splat (i8 1)
-; SSE4-NEXT: [[TMP58:%.*]] = add nuw <2 x i8> [[TMP57]], [[TMP56]]
-; SSE4-NEXT: [[TMP59:%.*]] = or <2 x i8> [[TMP25]], [[TMP20]]
-; SSE4-NEXT: [[TMP60:%.*]] = and <2 x i8> [[TMP59]], splat (i8 1)
-; SSE4-NEXT: [[TMP61:%.*]] = add nuw <2 x i8> [[TMP58]], [[TMP60]]
-; SSE4-NEXT: [[TMP62:%.*]] = insertelement <2 x i8> poison, i8 [[A_SROA_2_0_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP63:%.*]] = insertelement <2 x i8> [[TMP62]], i8 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP64:%.*]] = lshr <2 x i8> [[TMP63]], splat (i8 1)
-; SSE4-NEXT: [[TMP65:%.*]] = insertelement <2 x i8> poison, i8 [[A_SROA_16_8_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP66:%.*]] = insertelement <2 x i8> [[TMP65]], i8 [[B_SROA_16_8_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP67:%.*]] = lshr <2 x i8> [[TMP66]], splat (i8 1)
-; SSE4-NEXT: [[TMP68:%.*]] = add nuw <2 x i8> [[TMP67]], [[TMP64]]
-; SSE4-NEXT: [[TMP69:%.*]] = or <2 x i8> [[TMP66]], [[TMP63]]
-; SSE4-NEXT: [[TMP70:%.*]] = and <2 x i8> [[TMP69]], splat (i8 1)
-; SSE4-NEXT: [[TMP71:%.*]] = add nuw <2 x i8> [[TMP68]], [[TMP70]]
-; SSE4-NEXT: [[TMP72:%.*]] = insertelement <2 x i8> poison, i8 [[B_SROA_8_0_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP73:%.*]] = insertelement <2 x i8> [[TMP72]], i8 [[A_SROA_9_8_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP74:%.*]] = lshr <2 x i8> [[TMP73]], splat (i8 1)
-; SSE4-NEXT: [[TMP75:%.*]] = insertelement <2 x i8> poison, i8 [[B_SROA_17_8_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP76:%.*]] = insertelement <2 x i8> [[TMP75]], i8 [[B_SROA_17_8_EXTRACT_TRUNC1]], i64 1
-; SSE4-NEXT: [[TMP77:%.*]] = lshr <2 x i8> [[TMP76]], splat (i8 1)
-; SSE4-NEXT: [[TMP78:%.*]] = add nuw <2 x i8> [[TMP77]], [[TMP74]]
-; SSE4-NEXT: [[TMP79:%.*]] = or <2 x i8> [[TMP76]], [[TMP73]]
-; SSE4-NEXT: [[TMP80:%.*]] = and <2 x i8> [[TMP79]], splat (i8 1)
-; SSE4-NEXT: [[TMP81:%.*]] = add nuw <2 x i8> [[TMP78]], [[TMP80]]
-; SSE4-NEXT: [[TMP82:%.*]] = zext <2 x i8> [[TMP81]] to <2 x i64>
-; SSE4-NEXT: [[TMP83:%.*]] = shl nuw <2 x i64> [[TMP82]], splat (i64 56)
-; SSE4-NEXT: [[TMP84:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i64>
-; SSE4-NEXT: [[TMP85:%.*]] = shl nuw nsw <2 x i64> [[TMP84]], splat (i64 48)
-; SSE4-NEXT: [[TMP86:%.*]] = or disjoint <2 x i64> [[TMP83]], [[TMP85]]
-; SSE4-NEXT: [[TMP87:%.*]] = zext <2 x i8> [[TMP61]] to <2 x i64>
-; SSE4-NEXT: [[TMP88:%.*]] = shl nuw nsw <2 x i64> [[TMP87]], splat (i64 40)
-; SSE4-NEXT: [[TMP89:%.*]] = or disjoint <2 x i64> [[TMP86]], [[TMP88]]
-; SSE4-NEXT: [[TMP90:%.*]] = zext <2 x i8> [[TMP55]] to <2 x i64>
-; SSE4-NEXT: [[TMP91:%.*]] = shl nuw nsw <2 x i64> [[TMP90]], splat (i64 32)
-; SSE4-NEXT: [[TMP92:%.*]] = or disjoint <2 x i64> [[TMP89]], [[TMP91]]
-; SSE4-NEXT: [[TMP93:%.*]] = zext <2 x i8> [[TMP104]] to <2 x i64>
-; SSE4-NEXT: [[TMP94:%.*]] = shl nuw nsw <2 x i64> [[TMP93]], splat (i64 24)
-; SSE4-NEXT: [[TMP95:%.*]] = or disjoint <2 x i64> [[TMP92]], [[TMP94]]
-; SSE4-NEXT: [[TMP96:%.*]] = zext <2 x i8> [[TMP43]] to <2 x i64>
-; SSE4-NEXT: [[TMP97:%.*]] = shl nuw nsw <2 x i64> [[TMP96]], splat (i64 16)
-; SSE4-NEXT: [[TMP98:%.*]] = zext <2 x i8> [[TMP37]] to <2 x i64>
-; SSE4-NEXT: [[TMP99:%.*]] = shl nuw nsw <2 x i64> [[TMP98]], splat (i64 8)
-; SSE4-NEXT: [[TMP100:%.*]] = or disjoint <2 x i64> [[TMP95]], [[TMP97]]
-; SSE4-NEXT: [[TMP101:%.*]] = zext <2 x i8> [[TMP31]] to <2 x i64>
-; SSE4-NEXT: [[TMP102:%.*]] = or <2 x i64> [[TMP100]], [[TMP99]]
-; SSE4-NEXT: [[TMP103:%.*]] = or <2 x i64> [[TMP102]], [[TMP101]]
-; SSE4-NEXT: [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP103]], i64 0
+; SSE4-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
+; SSE4-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[TMP0]], <8 x i64> poison, <8 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP2:%.*]] = lshr <8 x i64> [[TMP1]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[TMP3:%.*]] = trunc <8 x i64> [[TMP2]] to <8 x i8>
+; SSE4-NEXT: [[TMP4:%.*]] = insertelement <8 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
+; SSE4-NEXT: [[TMP5:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP6:%.*]] = lshr <8 x i64> [[TMP5]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[TMP7:%.*]] = trunc <8 x i64> [[TMP6]] to <8 x i8>
+; SSE4-NEXT: [[TMP8:%.*]] = lshr <8 x i8> [[TMP3]], splat (i8 1)
+; SSE4-NEXT: [[TMP9:%.*]] = lshr <8 x i8> [[TMP7]], splat (i8 1)
+; SSE4-NEXT: [[TMP10:%.*]] = add nuw <8 x i8> [[TMP9]], [[TMP8]]
+; SSE4-NEXT: [[TMP11:%.*]] = or <8 x i8> [[TMP7]], [[TMP3]]
+; SSE4-NEXT: [[TMP12:%.*]] = and <8 x i8> [[TMP11]], splat (i8 1)
+; SSE4-NEXT: [[TMP13:%.*]] = add nuw <8 x i8> [[TMP10]], [[TMP12]]
+; SSE4-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[TMP13]] to <8 x i64>
+; SSE4-NEXT: [[TMP15:%.*]] = shl nuw <8 x i64> [[TMP14]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP15]])
; SSE4-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT]], 0
-; SSE4-NEXT: [[TMP49:%.*]] = extractelement <2 x i64> [[TMP103]], i64 1
+; SSE4-NEXT: [[TMP17:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE1:%.*]], i64 0
+; SSE4-NEXT: [[TMP18:%.*]] = shufflevector <8 x i64> [[TMP17]], <8 x i64> poison, <8 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP19:%.*]] = lshr <8 x i64> [[TMP18]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[TMP20:%.*]] = trunc <8 x i64> [[TMP19]] to <8 x i8>
+; SSE4-NEXT: [[TMP21:%.*]] = insertelement <8 x i64> poison, i64 [[B_COERCE1:%.*]], i64 0
+; SSE4-NEXT: [[TMP22:%.*]] = shufflevector <8 x i64> [[TMP21]], <8 x i64> poison, <8 x i32> zeroinitializer
+; SSE4-NEXT: [[TMP23:%.*]] = lshr <8 x i64> [[TMP22]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[TMP24:%.*]] = trunc <8 x i64> [[TMP23]] to <8 x i8>
+; SSE4-NEXT: [[TMP25:%.*]] = lshr <8 x i8> [[TMP20]], splat (i8 1)
+; SSE4-NEXT: [[TMP26:%.*]] = lshr <8 x i8> [[TMP24]], splat (i8 1)
+; SSE4-NEXT: [[TMP27:%.*]] = add nuw <8 x i8> [[TMP26]], [[TMP25]]
+; SSE4-NEXT: [[TMP28:%.*]] = or <8 x i8> [[TMP24]], [[TMP20]]
+; SSE4-NEXT: [[TMP29:%.*]] = and <8 x i8> [[TMP28]], splat (i8 1)
+; SSE4-NEXT: [[TMP30:%.*]] = add nuw <8 x i8> [[TMP27]], [[TMP29]]
+; SSE4-NEXT: [[TMP31:%.*]] = zext <8 x i8> [[TMP30]] to <8 x i64>
+; SSE4-NEXT: [[TMP32:%.*]] = shl nuw <8 x i64> [[TMP31]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; SSE4-NEXT: [[TMP49:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP32]])
; SSE4-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP49]], 1
; SSE4-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
index 426043033da90..7112b055647af 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
@@ -346,38 +346,20 @@ define i64 @combined(ptr nocapture noundef readonly %src) {
;
; SSE4-LABEL: @combined(
; SSE4-NEXT: entry:
-; SSE4-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 2
-; SSE4-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[ARRAYIDX_2]], align 2
-; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <8 x i64> [[TMP0]], zeroinitializer
-; SSE4-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP0]], <8 x i64> <i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 poison, i64 poison>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 6, i32 7>
-; SSE4-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
-; SSE4-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10
-; SSE4-NEXT: [[TMP4:%.*]] = load <2 x i64>, ptr [[SRC]], align 2
-; SSE4-NEXT: [[TMP5:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_10]], align 2
-; SSE4-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE4-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[TMP8]], zeroinitializer
-; SSE4-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> <i64 1, i64 2, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 5, i32 2, i32 2>
-; SSE4-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP10]], <4 x i64> zeroinitializer
-; SSE4-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP3]])
-; SSE4-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP11]])
-; SSE4-NEXT: [[OP_RDX4:%.*]] = or i64 [[TMP13]], [[TMP12]]
+; SSE4-NEXT: [[TMP0:%.*]] = load <12 x i64>, ptr [[SRC:%.*]], align 2
+; SSE4-NEXT: [[TMP1:%.*]] = icmp ne <12 x i64> [[TMP0]], zeroinitializer
+; SSE4-NEXT: [[TMP2:%.*]] = shufflevector <12 x i64> [[TMP0]], <12 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 poison, i64 poison, i64 poison, i64 poison>, <12 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 10>
+; SSE4-NEXT: [[TMP3:%.*]] = select <12 x i1> [[TMP1]], <12 x i64> [[TMP2]], <12 x i64> zeroinitializer
+; SSE4-NEXT: [[OP_RDX4:%.*]] = call i64 @llvm.vector.reduce.or.v12i64(<12 x i64> [[TMP3]])
; SSE4-NEXT: ret i64 [[OP_RDX4]]
;
; AVX-LABEL: @combined(
; AVX-NEXT: entry:
-; AVX-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 2
-; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
-; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
-; AVX-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; AVX-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8
-; AVX-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_8]], align 2
-; AVX-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], zeroinitializer
-; AVX-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
-; AVX-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> [[TMP6]]
-; AVX-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]])
-; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP3]], [[TMP8]]
+; AVX-NEXT: [[TMP0:%.*]] = load <12 x i64>, ptr [[SRC:%.*]], align 2
+; AVX-NEXT: [[TMP1:%.*]] = icmp ne <12 x i64> [[TMP0]], zeroinitializer
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <12 x i64> [[TMP0]], <12 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 poison, i64 poison, i64 poison, i64 poison>, <12 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 10>
+; AVX-NEXT: [[TMP3:%.*]] = select <12 x i1> [[TMP1]], <12 x i64> [[TMP2]], <12 x i64> zeroinitializer
+; AVX-NEXT: [[OP_RDX:%.*]] = call i64 @llvm.vector.reduce.or.v12i64(<12 x i64> [[TMP3]])
; AVX-NEXT: ret i64 [[OP_RDX]]
;
; AVX512-LABEL: @combined(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
index a5962bc787e9e..66197fecd8406 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
@@ -4,26 +4,23 @@
define i32 @test() {
; CHECK-LABEL: define i32 @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr getelementptr inbounds nuw (i8, ptr null, i64 16), align 16
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr null, align 16
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[TMP12]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <6 x i64> [[TMP3]], <6 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 0, i64 poison>, <6 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 5>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <6 x i64> [[TMP4]], <6 x i64> [[TMP3]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
-; CHECK-NEXT: [[TMP6:%.*]] = trunc <6 x i64> [[TMP5]] to <6 x i32>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <6 x i32> [[TMP6]], <6 x i32> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <6 x i32> <i32 1, i32 2, i32 3, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP11:%.*]] = trunc <6 x i64> [[TMP10]] to <6 x i32>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <6 x i32> [[TMP11]], <6 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 2, i32 2, i32 5, i32 4, i32 3>
-; CHECK-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP14]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = add <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison>, i32 [[TMP4]], i64 5
+; CHECK-NEXT: [[TMP6:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; CHECK-NEXT: [[TMP14:%.*]] = trunc <4 x i64> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP10]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> poison, <24 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7, i32 3, i32 2, i32 1>
+; CHECK-NEXT: [[TMP16:%.*]] = add <24 x i32> [[TMP15]], zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; CHECK-NEXT: [[INC_3_3_I_1:%.*]] = or i64 [[TMP9]], 0
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x i32> [[TMP8]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[RDX_OP:%.*]] = or <8 x i32> [[TMP16]], [[TMP15]]
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[RDX_OP]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x i32> [[TMP8]], <16 x i32> [[TMP18]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[OP_RDX:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP17]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = call i32 @llvm.vector.reduce.or.v24i32(<24 x i32> [[TMP16]])
; CHECK-NEXT: ret i32 [[OP_RDX]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-use-bitcasted-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-use-bitcasted-reduction.ll
index 9f24074dd0f00..2f45ac2fdd09b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-use-bitcasted-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-use-bitcasted-reduction.ll
@@ -78,38 +78,34 @@ define i32 @test_multi_cond_use(i32 %arg, i32 %arg1, i1 %arg4, i1 %arg5) {
; CHECK-NEXT: [[BB:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[ARG]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ARG1]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = sdiv <4 x i32> [[TMP2]], splat (i32 64)
-; CHECK-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 0>
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 1, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <4 x i32> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP10]], i64 2
-; CHECK-NEXT: [[EXT:%.*]] = zext i1 [[TMP8]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> [[TMP8]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> <i32 1, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 2, i32 4, i32 8>
; CHECK-NEXT: [[ICMP7:%.*]] = extractelement <4 x i1> [[TMP10]], i64 3
-; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[ICMP7]], i32 0, i32 2
-; CHECK-NEXT: [[ICMP9:%.*]] = extractelement <4 x i1> [[TMP10]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[ICMP9]], i32 0, i32 4
-; CHECK-NEXT: [[ICMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[ICMP11]], i32 0, i32 8
-; CHECK-NEXT: [[EXT1:%.*]] = zext i1 [[ICMP11]] to i32
+; CHECK-NEXT: [[EXT1:%.*]] = zext i1 [[ICMP7]] to i32
; CHECK-NEXT: br i1 [[ARG4]], label %[[BB13:.*]], label %[[BB16:.*]]
; CHECK: [[COMMON_RET:.*]]:
; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ [[EXT1]], %[[BB20:.*]] ], [ [[OR19:%.*]], %[[BB17:.*]] ]
; CHECK-NEXT: ret i32 [[COMMON_RET_OP]]
; CHECK: [[BB13]]:
-; CHECK-NEXT: [[OR:%.*]] = or i32 [[SELECT]], [[EXT]]
-; CHECK-NEXT: [[OR14:%.*]] = or i32 [[OR]], [[TMP12]]
-; CHECK-NEXT: [[TMP9:%.*]] = or i32 [[OR14]], [[TMP13]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP11]])
; CHECK-NEXT: ret i32 [[TMP9]]
; CHECK: [[BB16]]:
; CHECK-NEXT: br i1 [[ARG5]], label %[[BB17]], label %[[BB20]]
; CHECK: [[BB17]]:
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x i32> [[TMP11]], i64 0
+; CHECK-NEXT: [[SELECT:%.*]] = extractelement <4 x i32> [[TMP11]], i64 1
; CHECK-NEXT: [[OR18:%.*]] = or i32 [[SELECT]], [[EXT]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP11]], i64 2
; CHECK-NEXT: [[OR19]] = or i32 [[OR18]], [[TMP12]]
; CHECK-NEXT: br label %[[COMMON_RET]]
; CHECK: [[BB20]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP11]], i64 3
; CHECK-NEXT: store volatile i32 [[TMP13]], ptr null, align 4294967296
; CHECK-NEXT: br label %[[COMMON_RET]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
index d6b37914397f1..26a282baf2f6d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
@@ -8,25 +8,15 @@ define void @test(i32 %arg) {
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[ARG]], i64 0
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP5:%.*]], %[[BB1]] ]
-; CHECK-NEXT: [[PHI2:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP6:%.*]], %[[BB1]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[XOR27:%.*]], %[[BB1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP8:%.*]], %[[BB1]] ]
-; CHECK-NEXT: [[ADD14:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[ADD20:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[ADD5:%.*]] = add i32 [[PHI]], 0
-; CHECK-NEXT: [[ADD21:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = phi <12 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP6:%.*]], %[[BB1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add <12 x i32> [[TMP1]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP0]], <i32 0, i32 1>
-; CHECK-NEXT: [[TMP8]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-; CHECK-NEXT: [[TMP5]] = extractelement <8 x i32> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP6]] = extractelement <8 x i32> [[TMP8]], i64 2
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> [[TMP3]])
-; CHECK-NEXT: [[ADD23:%.*]] = xor i32 [[TMP7]], [[ADD14]]
-; CHECK-NEXT: [[XOR23:%.*]] = xor i32 [[ADD20]], [[ADD21]]
-; CHECK-NEXT: [[ADD26:%.*]] = xor i32 [[ADD5]], [[TMP6]]
-; CHECK-NEXT: [[XOR28:%.*]] = xor i32 [[ADD23]], [[XOR23]]
+; CHECK-NEXT: [[TMP6]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <12 x i32> <i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[ADD26:%.*]] = extractelement <12 x i32> [[TMP6]], i64 3
+; CHECK-NEXT: [[XOR28:%.*]] = call i32 @llvm.vector.reduce.xor.v12i32(<12 x i32> [[TMP2]])
; CHECK-NEXT: [[XOR27]] = xor i32 [[XOR28]], [[ADD26]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <12 x i32> [[TMP6]], i64 0
; CHECK-NEXT: [[ICMP:%.*]] = icmp ult i32 [[TMP5]], 0
; CHECK-NEXT: br label %[[BB1]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/replaced-external-in-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/replaced-external-in-reduction.ll
index 37bf890866f2e..4323833844b8f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/replaced-external-in-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/replaced-external-in-reduction.ll
@@ -5,23 +5,17 @@ define void @test(i32 %0, ptr %p) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i32 [[TMP0:%.*]], ptr [[P:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 0, i32 poison, i32 1, i32 1, i32 1, i32 poison>, i32 [[TMP0]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3>
-; CHECK-NEXT: [[TMP8:%.*]] = xor <8 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0>
-; CHECK-NEXT: [[OP_RDX:%.*]] = extractelement <8 x i32> [[TMP8]], i64 3
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <12 x i32> <i32 0, i32 0, i32 0, i32 poison, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP0]], i64 3
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <12 x i32> [[TMP1]], <12 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 3, i32 3, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP3:%.*]] = xor <12 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[OP_RDX:%.*]] = xor i32 [[TMP0]], 0
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[PH:%.*]]
; CHECK: ph:
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 0, i32 0, i32 0>, i32 [[TMP0]], i64 4
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <12 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, i32 [[TMP0]], i64 4
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP8]], [[ENTRY:%.*]] ], [ [[TMP6]], [[PH]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ [[TMP5]], [[ENTRY]] ], [ zeroinitializer, [[PH]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i32> [[TMP10]], [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[RDX_OP]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP12]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[OP_RDX5:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP11]])
+; CHECK-NEXT: [[TMP5:%.*]] = phi <12 x i32> [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TMP4]], [[PH]] ]
+; CHECK-NEXT: [[OP_RDX5:%.*]] = call i32 @llvm.vector.reduce.or.v12i32(<12 x i32> [[TMP5]])
; CHECK-NEXT: [[OP_RDX2:%.*]] = or i32 [[OP_RDX5]], [[OP_RDX]]
; CHECK-NEXT: store i32 [[OP_RDX2]], ptr [[P]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll b/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
index 3e9b1563f1a50..0708f24c238f2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/same-values-sub-node-with-poisons.ll
@@ -13,7 +13,7 @@ define i32 @test(ptr %f, i1 %tobool.i.4, i32 %retval.0.i.219) {
; CHECK-NEXT: br i1 false, label %[[D_EXIT_3]], label %[[D_EXIT_6:.*]]
; CHECK: [[D_EXIT_3]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[ENTRY]] ], [ poison, %[[IF_END_I_1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[RETVAL_0_I_219]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 1, i32 0>, i32 [[RETVAL_0_I_219]], i64 0
; CHECK-NEXT: br i1 [[TOBOOL_I_4]], label %[[D_EXIT_4:.*]], label %[[D_EXIT_6]]
; CHECK: [[D_EXIT_4]]:
; CHECK-NEXT: br label %[[D_EXIT_6]]
@@ -21,30 +21,22 @@ define i32 @test(ptr %f, i1 %tobool.i.4, i32 %retval.0.i.219) {
; CHECK-NEXT: br i1 false, label %[[D_EXIT_6]], label %[[D_EXIT_7:.*]]
; CHECK: [[D_EXIT_6]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP1]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ [[TMP1]], %[[D_EXIT_4]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP2]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[IF_END_I_5]] ], [ zeroinitializer, %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ poison, %[[IF_END_I_5]] ], [ [[TMP2]], %[[D_EXIT_3]] ], [ poison, %[[IF_END_I_2]] ], [ zeroinitializer, %[[D_EXIT_4]] ]
; CHECK-NEXT: br label %[[D_EXIT_7]]
; CHECK: [[D_EXIT_7]]:
; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ [[TMP3]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ [[TMP4]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i32> [ [[TMP13]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[RETVAL_0_I_219]], i64 3
-; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP15]], [[TMP9]]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP4]], %[[D_EXIT_6]] ], [ poison, %[[IF_END_I_5]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 1, i32 1, i32 3, i32 3>
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <8 x i32> [[TMP21]], <8 x i32> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP23]], <8 x i32> <i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 1>, <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 12, i32 5, i32 6, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP18]], [[TMP22]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i32> [[TMP20]], [[TMP16]]
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[RDX_OP]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP19]], <8 x i32> [[TMP25]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP12]])
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <12 x i32> [[TMP9]], <12 x i32> [[TMP10]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <12 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <12 x i32> [[TMP12]], <12 x i32> <i32 poison, i32 poison, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 1, i32 poison, i32 1, i32 1, i32 poison>, <12 x i32> <i32 0, i32 1, i32 14, i32 15, i32 16, i32 5, i32 6, i32 19, i32 poison, i32 21, i32 22, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <12 x i32> [[TMP13]], i32 [[TMP0]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <12 x i32> [[TMP18]], i32 [[RETVAL_0_I_219]], i64 11
+; CHECK-NEXT: [[TMP16:%.*]] = add <12 x i32> [[TMP15]], [[TMP11]]
+; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v12i32(<12 x i32> [[TMP16]])
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
diff --git a/llvm/test/Transforms/VectorCombine/X86/fold-equivalent-reduction-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/fold-equivalent-reduction-cmp.ll
index 9f11651e04f4c..b76edbcd0ca70 100644
--- a/llvm/test/Transforms/VectorCombine/X86/fold-equivalent-reduction-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/fold-equivalent-reduction-cmp.ll
@@ -16,8 +16,8 @@ define i1 @or_eq_0(<4 x i32> %x) {
define i1 @or_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[X]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[RED]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
%red = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %x)
@@ -28,7 +28,7 @@ define i1 @or_ne_0(<4 x i32> %x) {
define i1 @or_slt_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_slt_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[RED]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -40,7 +40,7 @@ define i1 @or_slt_0(<4 x i32> %x) {
define i1 @or_sgt_m1(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_sgt_m1(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[RED]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -52,7 +52,7 @@ define i1 @or_sgt_m1(<4 x i32> %x) {
define i1 @and_eq_allones(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_eq_allones(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -64,7 +64,7 @@ define i1 @and_eq_allones(<4 x i32> %x) {
define i1 @and_ne_allones(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_ne_allones(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[RED]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -76,7 +76,7 @@ define i1 @and_ne_allones(<4 x i32> %x) {
define i1 @and_slt_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_slt_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[RED]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -88,7 +88,7 @@ define i1 @and_slt_0(<4 x i32> %x) {
define i1 @and_sgt_m1(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_sgt_m1(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[RED]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -232,7 +232,7 @@ define i1 @umax_eq_0_i8(<16 x i8> %x) {
define i1 @umin_eq_allones_i64(<2 x i64> %x) {
; CHECK-LABEL: define i1 @umin_eq_allones_i64(
; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
diff --git a/llvm/test/Transforms/VectorCombine/X86/fold-reduction-zero-test.ll b/llvm/test/Transforms/VectorCombine/X86/fold-reduction-zero-test.ll
index 176bcff8c10cb..31e31ae948bb8 100644
--- a/llvm/test/Transforms/VectorCombine/X86/fold-reduction-zero-test.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/fold-reduction-zero-test.ll
@@ -1,18 +1,30 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,V1
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,V2
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,V3
-; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,V4
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,AVX512
declare i8 @llvm.vector.reduce.or.v16i8(<16 x i8>)
declare i8 @llvm.vector.reduce.umax.v16i8(<16 x i8>)
define i1 @or_nonzero(<16 x i8> %x) {
-; CHECK-LABEL: define i1 @or_nonzero(
-; CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
-; CHECK-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @or_nonzero(
+; SSE-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; SSE-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
+;
+; AVX2-LABEL: define i1 @or_nonzero(
+; AVX2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; AVX2-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
+; AVX2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; AVX2-NEXT: ret i1 [[CMP]]
+;
+; AVX512-LABEL: define i1 @or_nonzero(
+; AVX512-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; AVX512-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[X]])
+; AVX512-NEXT: [[CMP:%.*]] = icmp ne i8 [[REDUCTION]], 0
+; AVX512-NEXT: ret i1 [[CMP]]
;
%reduction = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %x)
%cmp = icmp ne i8 %reduction, 0
@@ -20,11 +32,23 @@ define i1 @or_nonzero(<16 x i8> %x) {
}
define i1 @or_zero(<16 x i8> %x) {
-; CHECK-LABEL: define i1 @or_zero(
-; CHECK-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
-; CHECK-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @or_zero(
+; SSE-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
+;
+; AVX2-LABEL: define i1 @or_zero(
+; AVX2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX2-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
+; AVX2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
+; AVX2-NEXT: ret i1 [[CMP]]
+;
+; AVX512-LABEL: define i1 @or_zero(
+; AVX512-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX512-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[X]])
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i8 [[REDUCTION]], 0
+; AVX512-NEXT: ret i1 [[CMP]]
;
%reduction = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %x)
%cmp = icmp eq i8 %reduction, 0
@@ -32,29 +56,23 @@ define i1 @or_zero(<16 x i8> %x) {
}
define i1 @umax_nonzero(<16 x i8> %x) {
-; V1-LABEL: define i1 @umax_nonzero(
-; V1-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V1-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
-; V1-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
-; V1-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @umax_nonzero(
+; SSE-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; SSE-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
-; V2-LABEL: define i1 @umax_nonzero(
-; V2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V2-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
-; V2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
-; V2-NEXT: ret i1 [[CMP]]
+; AVX2-LABEL: define i1 @umax_nonzero(
+; AVX2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX2-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
+; AVX2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; AVX2-NEXT: ret i1 [[CMP]]
;
-; V3-LABEL: define i1 @umax_nonzero(
-; V3-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V3-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[X]], zeroinitializer
-; V3-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
-; V3-NEXT: ret i1 [[CMP]]
-;
-; V4-LABEL: define i1 @umax_nonzero(
-; V4-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V4-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> [[X]])
-; V4-NEXT: [[CMP:%.*]] = icmp ne i8 [[REDUCTION]], 0
-; V4-NEXT: ret i1 [[CMP]]
+; AVX512-LABEL: define i1 @umax_nonzero(
+; AVX512-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX512-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> [[X]])
+; AVX512-NEXT: [[CMP:%.*]] = icmp ne i8 [[REDUCTION]], 0
+; AVX512-NEXT: ret i1 [[CMP]]
;
%reduction = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %x)
%cmp = icmp ne i8 %reduction, 0
@@ -62,29 +80,23 @@ define i1 @umax_nonzero(<16 x i8> %x) {
}
define i1 @umax_zero_commuted(<16 x i8> %x) {
-; V1-LABEL: define i1 @umax_zero_commuted(
-; V1-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V1-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
-; V1-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
-; V1-NEXT: ret i1 [[CMP]]
-;
-; V2-LABEL: define i1 @umax_zero_commuted(
-; V2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V2-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
-; V2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
-; V2-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @umax_zero_commuted(
+; SSE-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
-; V3-LABEL: define i1 @umax_zero_commuted(
-; V3-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V3-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
-; V3-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
-; V3-NEXT: ret i1 [[CMP]]
+; AVX2-LABEL: define i1 @umax_zero_commuted(
+; AVX2-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX2-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[X]], zeroinitializer
+; AVX2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
+; AVX2-NEXT: ret i1 [[CMP]]
;
-; V4-LABEL: define i1 @umax_zero_commuted(
-; V4-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
-; V4-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> [[X]])
-; V4-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[REDUCTION]]
-; V4-NEXT: ret i1 [[CMP]]
+; AVX512-LABEL: define i1 @umax_zero_commuted(
+; AVX512-SAME: <16 x i8> [[X:%.*]]) #[[ATTR1]] {
+; AVX512-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> [[X]])
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[REDUCTION]]
+; AVX512-NEXT: ret i1 [[CMP]]
;
%reduction = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %x)
%cmp = icmp eq i8 0, %reduction
@@ -93,7 +105,7 @@ define i1 @umax_zero_commuted(<16 x i8> %x) {
define i1 @or_nonzero_multiuse(<16 x i8> %x, ptr %out) {
; CHECK-LABEL: define i1 @or_nonzero_multiuse(
-; CHECK-SAME: <16 x i8> [[X:%.*]], ptr [[OUT:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: <16 x i8> [[X:%.*]], ptr [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[REDUCTION:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[X]])
; CHECK-NEXT: store i8 [[REDUCTION]], ptr [[OUT]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[REDUCTION]], 0
diff --git a/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp-codesize.ll b/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp-codesize.ll
index 1d3bb36873109..a7631106be188 100644
--- a/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp-codesize.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp-codesize.ll
@@ -12,7 +12,7 @@
define i1 @throughput(<4 x i32> %x) {
; CHECK-LABEL: define i1 @throughput(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
diff --git a/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp.ll
index a7bbe290d8750..25cfb4aeaa7e3 100644
--- a/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/fold-signbit-reduction-cmp.ll
@@ -7,7 +7,7 @@
define i1 @or_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -20,7 +20,7 @@ define i1 @or_eq_0(<4 x i32> %x) {
define i1 @or_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -33,7 +33,7 @@ define i1 @or_ne_0(<4 x i32> %x) {
define i1 @or_eq_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_eq_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -46,7 +46,7 @@ define i1 @or_eq_max(<4 x i32> %x) {
define i1 @or_ne_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @or_ne_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -59,7 +59,7 @@ define i1 @or_ne_max(<4 x i32> %x) {
define i1 @umax_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umax_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -72,7 +72,7 @@ define i1 @umax_eq_0(<4 x i32> %x) {
define i1 @umax_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umax_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -85,7 +85,7 @@ define i1 @umax_ne_0(<4 x i32> %x) {
define i1 @umax_eq_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umax_eq_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -98,7 +98,7 @@ define i1 @umax_eq_max(<4 x i32> %x) {
define i1 @umax_ne_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umax_ne_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -111,7 +111,7 @@ define i1 @umax_ne_max(<4 x i32> %x) {
define i1 @and_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -124,7 +124,7 @@ define i1 @and_eq_0(<4 x i32> %x) {
define i1 @and_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -137,7 +137,7 @@ define i1 @and_ne_0(<4 x i32> %x) {
define i1 @and_eq_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_eq_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -150,7 +150,7 @@ define i1 @and_eq_max(<4 x i32> %x) {
define i1 @and_ne_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @and_ne_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -163,7 +163,7 @@ define i1 @and_ne_max(<4 x i32> %x) {
define i1 @umin_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umin_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -176,7 +176,7 @@ define i1 @umin_eq_0(<4 x i32> %x) {
define i1 @umin_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umin_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -189,7 +189,7 @@ define i1 @umin_ne_0(<4 x i32> %x) {
define i1 @umin_eq_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umin_eq_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -202,7 +202,7 @@ define i1 @umin_eq_max(<4 x i32> %x) {
define i1 @umin_ne_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @umin_ne_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -215,7 +215,7 @@ define i1 @umin_ne_max(<4 x i32> %x) {
define i1 @add_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -228,7 +228,7 @@ define i1 @add_eq_0(<4 x i32> %x) {
define i1 @add_ne_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_ne_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -241,7 +241,7 @@ define i1 @add_ne_0(<4 x i32> %x) {
define i1 @add_eq_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_eq_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -254,7 +254,7 @@ define i1 @add_eq_max(<4 x i32> %x) {
define i1 @add_ne_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_ne_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -267,7 +267,7 @@ define i1 @add_ne_max(<4 x i32> %x) {
define i1 @add_ult_max(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_ult_max(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -280,7 +280,7 @@ define i1 @add_ult_max(<4 x i32> %x) {
define i1 @add_ugt_max_minus_1(<4 x i32> %x) {
; CHECK-LABEL: define i1 @add_ugt_max_minus_1(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -293,7 +293,7 @@ define i1 @add_ugt_max_minus_1(<4 x i32> %x) {
define i1 @ashr_add_eq_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_eq_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -304,11 +304,23 @@ define i1 @ashr_add_eq_0(<4 x i32> %x) {
}
define i1 @or_eq_0_v8i16(<8 x i16> %x) {
-; CHECK-LABEL: define i1 @or_eq_0_v8i16(
-; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> [[X]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[TMP1]], -1
-; CHECK-NEXT: ret i1 [[CMP]]
+; SSE2-LABEL: define i1 @or_eq_0_v8i16(
+; SSE2-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; SSE2-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[X]])
+; SSE2-NEXT: [[CMP:%.*]] = icmp sgt i16 [[TMP1]], -1
+; SSE2-NEXT: ret i1 [[CMP]]
+;
+; SSE4-LABEL: define i1 @or_eq_0_v8i16(
+; SSE4-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; SSE4-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> [[X]])
+; SSE4-NEXT: [[CMP:%.*]] = icmp sgt i16 [[TMP1]], -1
+; SSE4-NEXT: ret i1 [[CMP]]
+;
+; AVX-LABEL: define i1 @or_eq_0_v8i16(
+; AVX-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; AVX-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[X]])
+; AVX-NEXT: [[CMP:%.*]] = icmp sgt i16 [[TMP1]], -1
+; AVX-NEXT: ret i1 [[CMP]]
;
%shr = lshr <8 x i16> %x, splat (i16 15)
%red = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %shr)
@@ -317,23 +329,11 @@ define i1 @or_eq_0_v8i16(<8 x i16> %x) {
}
define i1 @and_eq_max_v2i64(<2 x i64> %x) {
-; SSE-LABEL: define i1 @and_eq_max_v2i64(
-; SSE-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
-; SSE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> [[X]])
-; SSE-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP1]], 0
-; SSE-NEXT: ret i1 [[CMP]]
-;
-; AVX2-LABEL: define i1 @and_eq_max_v2i64(
-; AVX2-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> [[X]])
-; AVX2-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP1]], 0
-; AVX2-NEXT: ret i1 [[CMP]]
-;
-; AVX512-LABEL: define i1 @and_eq_max_v2i64(
-; AVX512-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
-; AVX512-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[X]])
-; AVX512-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP1]], 0
-; AVX512-NEXT: ret i1 [[CMP]]
+; CHECK-LABEL: define i1 @and_eq_max_v2i64(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> [[X]])
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[TMP1]], 0
+; CHECK-NEXT: ret i1 [[CMP]]
;
%shr = lshr <2 x i64> %x, splat (i64 63)
%red = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %shr)
@@ -343,19 +343,12 @@ define i1 @and_eq_max_v2i64(<2 x i64> %x) {
; negative: shift amount is not bitwidth-1
define i1 @negative_wrong_shift(<4 x i32> %x) {
-; SSE2-LABEL: define i1 @negative_wrong_shift(
-; SSE2-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; SSE2-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 30)
-; SSE2-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[SHR]])
-; SSE2-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
-; SSE2-NEXT: ret i1 [[CMP]]
-;
-; SSE4-LABEL: define i1 @negative_wrong_shift(
-; SSE4-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; SSE4-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 30)
-; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SHR]], zeroinitializer
-; SSE4-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
-; SSE4-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @negative_wrong_shift(
+; SSE-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 30)
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SHR]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
; AVX2-LABEL: define i1 @negative_wrong_shift(
; AVX2-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
@@ -367,7 +360,7 @@ define i1 @negative_wrong_shift(<4 x i32> %x) {
; AVX512-LABEL: define i1 @negative_wrong_shift(
; AVX512-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
; AVX512-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 30)
-; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[SHR]])
+; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[SHR]])
; AVX512-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
; AVX512-NEXT: ret i1 [[CMP]]
;
@@ -394,21 +387,13 @@ define i1 @negative_wrong_cmp_const(<4 x i32> %x) {
; negative: shift has multiple uses
define i1 @negative_multi_use_shift(<4 x i32> %x, ptr %p) {
-; SSE2-LABEL: define i1 @negative_multi_use_shift(
-; SSE2-SAME: <4 x i32> [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE2-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 31)
-; SSE2-NEXT: store <4 x i32> [[SHR]], ptr [[P]], align 16
-; SSE2-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[SHR]])
-; SSE2-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
-; SSE2-NEXT: ret i1 [[CMP]]
-;
-; SSE4-LABEL: define i1 @negative_multi_use_shift(
-; SSE4-SAME: <4 x i32> [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE4-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 31)
-; SSE4-NEXT: store <4 x i32> [[SHR]], ptr [[P]], align 16
-; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SHR]], zeroinitializer
-; SSE4-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
-; SSE4-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @negative_multi_use_shift(
+; SSE-SAME: <4 x i32> [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 31)
+; SSE-NEXT: store <4 x i32> [[SHR]], ptr [[P]], align 16
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SHR]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
; AVX2-LABEL: define i1 @negative_multi_use_shift(
; AVX2-SAME: <4 x i32> [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
@@ -422,7 +407,7 @@ define i1 @negative_multi_use_shift(<4 x i32> %x, ptr %p) {
; AVX512-SAME: <4 x i32> [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
; AVX512-NEXT: [[SHR:%.*]] = lshr <4 x i32> [[X]], splat (i32 31)
; AVX512-NEXT: store <4 x i32> [[SHR]], ptr [[P]], align 16
-; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[SHR]])
+; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[SHR]])
; AVX512-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
; AVX512-NEXT: ret i1 [[CMP]]
;
@@ -481,7 +466,7 @@ define i1 @negative_add_numelts_overflow(<8 x i2> %x) {
define i1 @ashr_add_eq_allneg(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_eq_allneg(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -494,7 +479,7 @@ define i1 @ashr_add_eq_allneg(<4 x i32> %x) {
define i1 @ashr_add_ne_allneg(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_ne_allneg(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -507,7 +492,7 @@ define i1 @ashr_add_ne_allneg(<4 x i32> %x) {
define i1 @ashr_add_sgt_minus1(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_sgt_minus1(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[RED]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -520,7 +505,7 @@ define i1 @ashr_add_sgt_minus1(<4 x i32> %x) {
define i1 @ashr_add_slt_0(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_slt_0(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[RED]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -533,7 +518,7 @@ define i1 @ashr_add_slt_0(<4 x i32> %x) {
define i1 @ashr_add_slt_minus3(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_slt_minus3(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -546,7 +531,7 @@ define i1 @ashr_add_slt_minus3(<4 x i32> %x) {
define i1 @ashr_add_sgt_minus4(<4 x i32> %x) {
; CHECK-LABEL: define i1 @ashr_add_sgt_minus4(
; CHECK-SAME: <4 x i32> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[X]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -557,11 +542,23 @@ define i1 @ashr_add_sgt_minus4(<4 x i32> %x) {
}
define i1 @ashr_add_eq_allneg_v8i16(<8 x i16> %x) {
-; CHECK-LABEL: define i1 @ashr_add_eq_allneg_v8i16(
-; CHECK-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[X]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[TMP1]], 0
-; CHECK-NEXT: ret i1 [[CMP]]
+; SSE2-LABEL: define i1 @ashr_add_eq_allneg_v8i16(
+; SSE2-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; SSE2-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> [[X]])
+; SSE2-NEXT: [[CMP:%.*]] = icmp slt i16 [[TMP1]], 0
+; SSE2-NEXT: ret i1 [[CMP]]
+;
+; SSE4-LABEL: define i1 @ashr_add_eq_allneg_v8i16(
+; SSE4-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; SSE4-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[X]])
+; SSE4-NEXT: [[CMP:%.*]] = icmp slt i16 [[TMP1]], 0
+; SSE4-NEXT: ret i1 [[CMP]]
+;
+; AVX-LABEL: define i1 @ashr_add_eq_allneg_v8i16(
+; AVX-SAME: <8 x i16> [[X:%.*]]) #[[ATTR0]] {
+; AVX-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> [[X]])
+; AVX-NEXT: [[CMP:%.*]] = icmp slt i16 [[TMP1]], 0
+; AVX-NEXT: ret i1 [[CMP]]
;
%shr = ashr <8 x i16> %x, splat (i16 15)
%red = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %shr)
@@ -825,7 +822,7 @@ define i1 @multi_add_lshr_eq_0(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_lshr_eq_0(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -841,7 +838,7 @@ define i1 @multi_add_lshr_ne_0(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_lshr_ne_0(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -857,7 +854,7 @@ define i1 @multi_add_lshr_eq_8(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_lshr_eq_8(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -873,7 +870,7 @@ define i1 @multi_add_lshr_ne_8(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_lshr_ne_8(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -889,7 +886,7 @@ define i1 @multi_or_lshr_eq_0(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_or_lshr_eq_0(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -905,7 +902,7 @@ define i1 @multi_and_lshr_eq_1(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_and_lshr_eq_1(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -922,7 +919,7 @@ define i1 @multi_triple_add_lshr_eq_0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[C]], [[B]]
; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[A]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP3]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -941,7 +938,7 @@ define i1 @multi_triple_add_lshr_eq_12(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[C]], [[B]]
; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[TMP1]], [[A]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP3]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -959,7 +956,7 @@ define i1 @multi_add_ashr_eq_0(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_ashr_eq_0(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -975,7 +972,7 @@ define i1 @multi_add_ashr_eq_minus8(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_ashr_eq_minus8(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -991,7 +988,7 @@ define i1 @multi_add_ashr_sgt_minus1(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_ashr_sgt_minus1(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -1007,7 +1004,7 @@ define i1 @multi_add_ashr_slt_minus7(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_add_ashr_slt_minus7(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -1023,7 +1020,7 @@ define i1 @multi_umax_or_tree_eq_0(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_umax_or_tree_eq_0(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP2]], -1
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -1039,7 +1036,7 @@ define i1 @multi_umin_and_tree_eq_1(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: define i1 @multi_umin_and_tree_eq_1(
; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[B]], [[A]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -1091,25 +1088,15 @@ define i1 @negative_multi_overflow(<8 x i2> %a, <8 x i2> %b) {
; negative: shift has multiple uses
define i1 @negative_multi_shift_multiuse(<4 x i32> %a, <4 x i32> %b, ptr %p) {
-; SSE2-LABEL: define i1 @negative_multi_shift_multiuse(
-; SSE2-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE2-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
-; SSE2-NEXT: store <4 x i32> [[SA]], ptr [[P]], align 16
-; SSE2-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
-; SSE2-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; SSE2-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[SUM]])
-; SSE2-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
-; SSE2-NEXT: ret i1 [[CMP]]
-;
-; SSE4-LABEL: define i1 @negative_multi_shift_multiuse(
-; SSE4-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE4-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
-; SSE4-NEXT: store <4 x i32> [[SA]], ptr [[P]], align 16
-; SSE4-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
-; SSE4-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SUM]], zeroinitializer
-; SSE4-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
-; SSE4-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @negative_multi_shift_multiuse(
+; SSE-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
+; SSE-NEXT: store <4 x i32> [[SA]], ptr [[P]], align 16
+; SSE-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
+; SSE-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SUM]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
; AVX2-LABEL: define i1 @negative_multi_shift_multiuse(
; AVX2-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
@@ -1127,7 +1114,7 @@ define i1 @negative_multi_shift_multiuse(<4 x i32> %a, <4 x i32> %b, ptr %p) {
; AVX512-NEXT: store <4 x i32> [[SA]], ptr [[P]], align 16
; AVX512-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
; AVX512-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[SUM]])
+; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[SUM]])
; AVX512-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
; AVX512-NEXT: ret i1 [[CMP]]
;
@@ -1142,29 +1129,17 @@ define i1 @negative_multi_shift_multiuse(<4 x i32> %a, <4 x i32> %b, ptr %p) {
; negative: internal tree node has multiple uses (cost driven)
define i1 @negative_multi_tree_node_multiuse(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, ptr %p) {
-; SSE2-LABEL: define i1 @negative_multi_tree_node_multiuse(
-; SSE2-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE2-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
-; SSE2-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
-; SSE2-NEXT: [[SC:%.*]] = lshr <4 x i32> [[C]], splat (i32 31)
-; SSE2-NEXT: [[AB:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; SSE2-NEXT: store <4 x i32> [[AB]], ptr [[P]], align 16
-; SSE2-NEXT: [[ABC:%.*]] = add <4 x i32> [[AB]], [[SC]]
-; SSE2-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ABC]])
-; SSE2-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
-; SSE2-NEXT: ret i1 [[CMP]]
-;
-; SSE4-LABEL: define i1 @negative_multi_tree_node_multiuse(
-; SSE4-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; SSE4-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
-; SSE4-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
-; SSE4-NEXT: [[SC:%.*]] = lshr <4 x i32> [[C]], splat (i32 31)
-; SSE4-NEXT: [[AB:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; SSE4-NEXT: store <4 x i32> [[AB]], ptr [[P]], align 16
-; SSE4-NEXT: [[ABC:%.*]] = add <4 x i32> [[AB]], [[SC]]
-; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[ABC]], zeroinitializer
-; SSE4-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
-; SSE4-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @negative_multi_tree_node_multiuse(
+; SSE-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
+; SSE-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
+; SSE-NEXT: [[SC:%.*]] = lshr <4 x i32> [[C]], splat (i32 31)
+; SSE-NEXT: [[AB:%.*]] = add <4 x i32> [[SA]], [[SB]]
+; SSE-NEXT: store <4 x i32> [[AB]], ptr [[P]], align 16
+; SSE-NEXT: [[ABC:%.*]] = add <4 x i32> [[AB]], [[SC]]
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[ABC]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
;
; AVX2-LABEL: define i1 @negative_multi_tree_node_multiuse(
; AVX2-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
@@ -1186,7 +1161,7 @@ define i1 @negative_multi_tree_node_multiuse(<4 x i32> %a, <4 x i32> %b, <4 x i3
; AVX512-NEXT: [[AB:%.*]] = add <4 x i32> [[SA]], [[SB]]
; AVX512-NEXT: store <4 x i32> [[AB]], ptr [[P]], align 16
; AVX512-NEXT: [[ABC:%.*]] = add <4 x i32> [[AB]], [[SC]]
-; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[ABC]])
+; AVX512-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[ABC]])
; AVX512-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
; AVX512-NEXT: ret i1 [[CMP]]
;
@@ -1203,14 +1178,32 @@ define i1 @negative_multi_tree_node_multiuse(<4 x i32> %a, <4 x i32> %b, <4 x i3
; negative: tree op (add) doesn't match reduction (or/umax expects or tree)
define i1 @negative_multi_op_mismatch(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: define i1 @negative_multi_op_mismatch(
-; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
-; CHECK-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
-; CHECK-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SUM]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
-; CHECK-NEXT: ret i1 [[CMP]]
+; SSE-LABEL: define i1 @negative_multi_op_mismatch(
+; SSE-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
+; SSE-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
+; SSE-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
+; SSE-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SUM]], zeroinitializer
+; SSE-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; SSE-NEXT: ret i1 [[CMP]]
+;
+; AVX2-LABEL: define i1 @negative_multi_op_mismatch(
+; AVX2-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
+; AVX2-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
+; AVX2-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
+; AVX2-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SUM]], zeroinitializer
+; AVX2-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
+; AVX2-NEXT: ret i1 [[CMP]]
+;
+; AVX512-LABEL: define i1 @negative_multi_op_mismatch(
+; AVX512-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT: [[SA:%.*]] = lshr <4 x i32> [[A]], splat (i32 31)
+; AVX512-NEXT: [[SB:%.*]] = lshr <4 x i32> [[B]], splat (i32 31)
+; AVX512-NEXT: [[SUM:%.*]] = add <4 x i32> [[SA]], [[SB]]
+; AVX512-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[SUM]])
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
+; AVX512-NEXT: ret i1 [[CMP]]
;
%sa = lshr <4 x i32> %a, splat (i32 31)
%sb = lshr <4 x i32> %b, splat (i32 31)
@@ -1229,7 +1222,7 @@ define i32 @PR209714(ptr %pa, ptr %pb) {
; CHECK-NEXT: [[A_NOT:%.*]] = xor <2 x i64> [[A]], splat (i64 -1)
; CHECK-NEXT: [[AND:%.*]] = and <2 x i64> [[B]], [[A_NOT]]
; CHECK-NEXT: [[AND_BC:%.*]] = bitcast <2 x i64> [[AND]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[AND_BC]])
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[AND_BC]])
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
; CHECK-NEXT: [[RES:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RES]]
@@ -1245,5 +1238,3 @@ define i32 @PR209714(ptr %pa, ptr %pb) {
%res = zext i1 %cmp to i32
ret i32 %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
diff --git a/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll b/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
index 28bc789236635..6aa99e4780ac1 100644
--- a/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
@@ -27,12 +27,10 @@ define i1 @or_zext_i3(<4 x i3> %x) {
ret i1 %cmp
}
-; negative: costly for X86
-define i1 @or_zext_v3_costly(<3 x i8> %x) {
-; CHECK-LABEL: define i1 @or_zext_v3_costly(
+define i1 @or_zext_v3(<3 x i8> %x) {
+; CHECK-LABEL: define i1 @or_zext_v3(
; CHECK-SAME: <3 x i8> [[X:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <3 x i8> [[X]] to <3 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i32> [[ZEXT]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i8> [[X]], zeroinitializer
; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v3i1(<3 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
@@ -45,9 +43,8 @@ define i1 @or_zext_v3_costly(<3 x i8> %x) {
define i1 @or_sext(<4 x i16> %x) {
; CHECK-LABEL: define i1 @or_sext(
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[X]])
-; CHECK-NEXT: [[RED:%.*]] = sext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%sext = sext <4 x i16> %x to <4 x i32>
@@ -431,9 +428,8 @@ define i1 @add_neg(<4 x i16> %x) {
define i1 @add_mul(<4 x i16> %x) {
; CHECK-LABEL: define i1 @add_mul(
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ZEXT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -446,9 +442,8 @@ define i1 @add_mul(<4 x i16> %x) {
define i1 @add_shl(<4 x i16> %x, <4 x i32> %y) {
; CHECK-LABEL: define i1 @add_shl(
; CHECK-SAME: <4 x i16> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ZEXT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -462,9 +457,8 @@ define i1 @add_shl(<4 x i16> %x, <4 x i32> %y) {
define i1 @add_shl_ne(<4 x i16> %x, <4 x i32> %y) {
; CHECK-LABEL: define i1 @add_shl_ne(
; CHECK-SAME: <4 x i16> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ZEXT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -519,8 +513,8 @@ define i1 @add_shl_multiuse(<4 x i16> %x, <4 x i32> %y, ptr %p) {
; CHECK-NEXT: [[YMASKED:%.*]] = and <4 x i32> [[Y]], splat (i32 7)
; CHECK-NEXT: [[SHL:%.*]] = shl nsw <4 x i32> [[ZEXT]], [[YMASKED]]
; CHECK-NEXT: store <4 x i32> [[SHL]], ptr [[P]], align 16
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[SHL]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[SHL]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -552,9 +546,8 @@ define i1 @add_shl_unbounded(<4 x i16> %x, <4 x i32> %y) {
define i1 @add_mul_nonsplat(<4 x i16> %x) {
; CHECK-LABEL: define i1 @add_mul_nonsplat(
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ZEXT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -567,9 +560,8 @@ define i1 @add_mul_nonsplat(<4 x i16> %x) {
define i1 @add_mul_poison(<4 x i16> %x) {
; CHECK-LABEL: define i1 @add_mul_poison(
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[ZEXT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -585,8 +577,8 @@ define i1 @add_mul_zero_lane(<4 x i16> %x) {
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
; CHECK-NEXT: [[ZEXT:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
; CHECK-NEXT: [[MUL:%.*]] = mul nuw <4 x i32> [[ZEXT]], <i32 1, i32 0, i32 3, i32 4>
-; CHECK-NEXT: [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[MUL]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[RED]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[MUL]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
%zext = zext <4 x i16> %x to <4 x i32>
@@ -620,7 +612,7 @@ define void @or_zext_two_blocks(<4 x i16> %x) {
; CHECK-SAME: <4 x i16> [[X:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[A:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <4 x i16> [[X]], zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <4 x i32> [[A]], zeroinitializer
; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP0]])
; CHECK-NEXT: br i1 [[CMP]], label %[[THEN:.*]], label %[[EXIT:.*]]
; CHECK: [[THEN]]:
>From dd09e2aabcf3b253986fd8d5c36f0881d88b73d3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Thu, 6 Aug 2026 11:38:34 +0100
Subject: [PATCH 2/2] Fix SSE2 baseline v16i8 reductions
---
llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 6 +++---
llvm/test/Analysis/CostModel/X86/reduce-and.ll | 14 +++++++-------
llvm/test/Analysis/CostModel/X86/reduce-or.ll | 14 +++++++-------
llvm/test/Analysis/CostModel/X86/reduce-xor.ll | 14 +++++++-------
.../VectorCombine/X86/icmp-vector-reduce.ll | 3 ++-
5 files changed, 26 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 4f424517e8d08..1053296ff408d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5723,15 +5723,15 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::AND, MVT::v2i64, {2, 2, 3, 3} },
{ ISD::AND, MVT::v4i32, {3, 4, 5, 5} },
{ ISD::AND, MVT::v8i16, {4, 7, 8, 8} },
- { ISD::AND, MVT::v16i8, {5,10,11,11} },
+ { ISD::AND, MVT::v16i8, {6,10,11,11} },
{ ISD::OR, MVT::v2i64, {2, 2, 3, 3} },
{ ISD::OR, MVT::v4i32, {3, 4, 5, 5} },
{ ISD::OR, MVT::v8i16, {4, 7, 8, 8} },
- { ISD::OR, MVT::v16i8, {5,10,11,11} },
+ { ISD::OR, MVT::v16i8, {6,10,11,11} },
{ ISD::XOR, MVT::v2i64, {2, 2, 3, 3} },
{ ISD::XOR, MVT::v4i32, {3, 4, 5, 5} },
{ ISD::XOR, MVT::v8i16, {4, 7, 8, 8} },
- { ISD::XOR, MVT::v16i8, {5,10,11,11} },
+ { ISD::XOR, MVT::v16i8, {6,10,11,11} },
};
static const CostKindTblEntry AVX1CostTbl[] = {
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-and.ll b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
index 72e10f07c0039..687bf7417c5c9 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-and.ll
@@ -138,13 +138,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:13 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.and.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-or.ll b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
index 86558a7fe85ba..c9b70aba869ce 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-or.ll
@@ -138,13 +138,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:13 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.or.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
index 7f6187cdf7092..8988e4307683c 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-xor.ll
@@ -138,13 +138,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; SSE-LABEL: 'reduce_i8'
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; SSE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V2 = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V4 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:11 Lat:10 SizeLat:11 for: %V16 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:12 Lat:11 SizeLat:12 for: %V32 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:14 Lat:13 SizeLat:14 for: %V64 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; SSE-NEXT: Cost Model: Found costs of RThru:13 CodeSize:18 Lat:17 SizeLat:18 for: %V128 = call i8 @llvm.vector.reduce.xor.v128i8(<128 x i8> undef)
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'reduce_i8'
diff --git a/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll b/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
index 6aa99e4780ac1..ced5633a8ec5d 100644
--- a/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/icmp-vector-reduce.ll
@@ -30,7 +30,8 @@ define i1 @or_zext_i3(<4 x i3> %x) {
define i1 @or_zext_v3(<3 x i8> %x) {
; CHECK-LABEL: define i1 @or_zext_v3(
; CHECK-SAME: <3 x i8> [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i8> [[X]], zeroinitializer
+; CHECK-NEXT: [[ZEXT:%.*]] = zext <3 x i8> [[X]] to <3 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <3 x i32> [[ZEXT]], zeroinitializer
; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v3i1(<3 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
More information about the llvm-commits
mailing list