[llvm] [TTI] Include scalarization overhead for icmp/fcmp result (PR #206697)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 08:09:58 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/206697
>From 79be917572d6ce3ef7ea4eaab82f70bdf72f008f Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 1 Jul 2026 13:03:34 +0800
Subject: [PATCH 1/7] Precommit tests
---
.../Analysis/CostModel/RISCV/fcmp-zve64x.ll | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
diff --git a/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll b/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
new file mode 100644
index 0000000000000..b98faa5bacea6
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -p 'print<cost-model>' -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
+
+define void @f() {
+; CHECK-LABEL: 'f'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = fcmp oge <4 x float> poison, poison
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = fcmp oge <4 x double> poison, poison
+; CHECK-NEXT: Cost Model: Invalid cost for instruction: %3 = fcmp oge <vscale x 2 x float> poison, poison
+; CHECK-NEXT: Cost Model: Invalid cost for instruction: %4 = fcmp oge <vscale x 2 x double> poison, poison
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ fcmp oge <4 x float> poison, poison
+ fcmp oge <4 x double> poison, poison
+ fcmp oge <vscale x 2 x float> poison, poison
+ fcmp oge <vscale x 2 x double> poison, poison
+ ret void
+}
>From 8efccea5ccb84ec8bbcbef170d7560942e0a9025 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 30 Jun 2026 18:02:08 +0800
Subject: [PATCH 2/7] [TTI] Include scalarization overhead for icmp/fcmp result
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 11 +-
llvm/test/Analysis/CostModel/AArch64/fshl.ll | 2 +-
llvm/test/Analysis/CostModel/AArch64/fshr.ll | 2 +-
.../CostModel/AMDGPU/arith-sminmax.ll | 56 +++---
.../CostModel/AMDGPU/arith-uminmax.ll | 56 +++---
.../CostModel/ARM/active_lane_mask.ll | 24 +--
.../Analysis/CostModel/ARM/arith-overflow.ll | 36 ++--
.../test/Analysis/CostModel/ARM/arith-ssat.ll | 96 +++++------
.../test/Analysis/CostModel/ARM/arith-usat.ll | 24 +--
llvm/test/Analysis/CostModel/ARM/cmps.ll | 42 ++---
llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll | 40 ++---
llvm/test/Analysis/CostModel/ARM/mve-abs.ll | 12 +-
llvm/test/Analysis/CostModel/ARM/mve-cmp.ll | 22 +--
.../test/Analysis/CostModel/ARM/mve-minmax.ll | 48 +++---
.../Analysis/CostModel/ARM/reduce-smax.ll | 58 +++----
.../Analysis/CostModel/ARM/reduce-smin.ll | 58 +++----
.../Analysis/CostModel/ARM/reduce-umax.ll | 58 +++----
.../Analysis/CostModel/ARM/reduce-umin.ll | 58 +++----
.../CostModel/PowerPC/cmp-expanded.ll | 2 +-
.../test/Analysis/CostModel/RISCV/cast-sat.ll | 80 ++++-----
.../Analysis/CostModel/RISCV/rvv-fcmp-f16.ll | 96 +++++------
llvm/test/Analysis/CostModel/X86/fptoi_sat.ll | 160 +++++++++---------
.../LoopVectorize/ARM/mve-icmpcost.ll | 8 +-
.../SLPVectorizer/AMDGPU/min_max.ll | 13 +-
.../SLPVectorizer/scalarazied-result.ll | 4 +
25 files changed, 539 insertions(+), 527 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 55163eb1976bf..cdd2d005af4aa 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1458,9 +1458,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// Return the cost of multiple scalar invocation plus the cost of
// inserting and extracting the values.
- return getScalarizationOverhead(ValVTy, /*Insert*/ true,
- /*Extract*/ false, CostKind) +
- Num * Cost;
+ InstructionCost Overhead =
+ getScalarizationOverhead(ValVTy, /*Insert*/ false,
+ /*Extract*/ true, CostKind);
+ if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
+ Overhead +=
+ getScalarizationOverhead(cast<VectorType>(CondTy), /*Insert*/ true,
+ /*Extract*/ false, CostKind);
+ return Overhead + Num * Cost;
}
// Unknown scalar opcode.
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshl.ll b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
index 4219a4de764f5..2264a77acdd92 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshl.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
@@ -286,7 +286,7 @@ entry:
define <2 x i128> @fshl_v2i128_3rd_arg_var(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-LABEL: 'fshl_v2i128_3rd_arg_var'
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:17 Lat:21 SizeLat:21 for: %r = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
+; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:19 Lat:25 SizeLat:25 for: %r = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %r
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshr.ll b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
index e3e84becd3e00..6c8c472268c49 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshr.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
@@ -286,7 +286,7 @@ entry:
define <2 x i128> @fshr_v2i128_3rd_arg_var(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-LABEL: 'fshr_v2i128_3rd_arg_var'
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:17 Lat:21 SizeLat:21 for: %r = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
+; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:19 Lat:25 SizeLat:25 for: %r = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %r
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/arith-sminmax.ll b/llvm/test/Analysis/CostModel/AMDGPU/arith-sminmax.ll
index 8afb26722cc16..80205342cbe52 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/arith-sminmax.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/arith-sminmax.ll
@@ -39,14 +39,14 @@ declare <64 x i8> @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
define i32 @smax(i32 %arg) {
; FAST-LABEL: 'smax'
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -64,14 +64,14 @@ define i32 @smax(i32 %arg) {
;
; SLOW-LABEL: 'smax'
; SLOW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -145,14 +145,14 @@ declare <64 x i8> @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
define i32 @smin(i32 %arg) {
; FAST-LABEL: 'smin'
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -170,14 +170,14 @@ define i32 @smin(i32 %arg) {
;
; SLOW-LABEL: 'smin'
; SLOW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/arith-uminmax.ll b/llvm/test/Analysis/CostModel/AMDGPU/arith-uminmax.ll
index 2b3728b556d9e..c5ca41dabd4d2 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/arith-uminmax.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/arith-uminmax.ll
@@ -39,14 +39,14 @@ declare <64 x i8> @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
define i32 @umax(i32 %arg) {
; FAST-LABEL: 'umax'
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -64,14 +64,14 @@ define i32 @umax(i32 %arg) {
;
; SLOW-LABEL: 'umax'
; SLOW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -145,14 +145,14 @@ declare <64 x i8> @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
define i32 @umin(i32 %arg) {
; FAST-LABEL: 'umin'
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; FAST-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; FAST-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; FAST-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
@@ -170,14 +170,14 @@ define i32 @umin(i32 %arg) {
;
; SLOW-LABEL: 'umin'
; SLOW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLOW-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLOW-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; SLOW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll b/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
index 38c43a5253a57..e0cf7ce9a7ea4 100644
--- a/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
+++ b/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
@@ -3,18 +3,18 @@
define void @get_lane_mask() {
; CHECK-LABEL: 'get_lane_mask'
-; CHECK-NEXT: Cost Model: Found costs of RThru:176 CodeSize:192 Lat:176 SizeLat:176 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:88 CodeSize:96 Lat:88 SizeLat:88 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:44 CodeSize:48 Lat:44 SizeLat:44 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:24 Lat:22 SizeLat:22 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:96 CodeSize:112 Lat:96 SizeLat:96 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:48 CodeSize:56 Lat:48 SizeLat:48 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:96 CodeSize:112 Lat:96 SizeLat:96 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:48 CodeSize:56 Lat:48 SizeLat:48 for: %mask_v8i1_i16 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %mask_v4i1_i16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %mask_v2i1_i16 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:192 CodeSize:208 Lat:192 SizeLat:192 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:96 CodeSize:104 Lat:96 SizeLat:96 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:48 CodeSize:52 Lat:48 SizeLat:48 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:24 CodeSize:26 Lat:24 SizeLat:24 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:112 CodeSize:128 Lat:112 SizeLat:112 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:56 CodeSize:64 Lat:56 SizeLat:56 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:28 CodeSize:32 Lat:28 SizeLat:28 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:16 Lat:14 SizeLat:14 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:112 CodeSize:128 Lat:112 SizeLat:112 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:56 CodeSize:64 Lat:56 SizeLat:56 for: %mask_v8i1_i16 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:28 CodeSize:32 Lat:28 SizeLat:28 for: %mask_v4i1_i16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:16 Lat:14 SizeLat:14 for: %mask_v2i1_i16 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i16(i16 undef, i16 undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: ret void
;
%mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
index 219b01dfc2dce..cd683d0c9cb43 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
@@ -66,9 +66,9 @@ define i32 @sadd(i32 %arg) {
;
; MVE-LABEL: 'sadd'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:92 Lat:126 SizeLat:126 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:218 CodeSize:145 Lat:218 SizeLat:218 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:434 CodeSize:289 Lat:434 SizeLat:434 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:108 Lat:158 SizeLat:158 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:282 CodeSize:177 Lat:282 SizeLat:282 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:562 CodeSize:353 Lat:562 SizeLat:562 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:78 CodeSize:71 Lat:78 SizeLat:78 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -167,9 +167,9 @@ define i32 @uadd(i32 %arg) {
;
; MVE-LABEL: 'uadd'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:54 Lat:72 SizeLat:72 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:144 CodeSize:108 Lat:144 SizeLat:144 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:288 CodeSize:216 Lat:288 SizeLat:288 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:62 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:124 Lat:176 SizeLat:176 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:352 CodeSize:248 Lat:352 SizeLat:352 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:36 Lat:40 SizeLat:40 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -268,9 +268,9 @@ define i32 @ssub(i32 %arg) {
;
; MVE-LABEL: 'ssub'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:92 Lat:126 SizeLat:126 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:218 CodeSize:145 Lat:218 SizeLat:218 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:434 CodeSize:289 Lat:434 SizeLat:434 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:108 Lat:158 SizeLat:158 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:282 CodeSize:177 Lat:282 SizeLat:282 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:562 CodeSize:353 Lat:562 SizeLat:562 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:78 CodeSize:71 Lat:78 SizeLat:78 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -369,9 +369,9 @@ define i32 @usub(i32 %arg) {
;
; MVE-LABEL: 'usub'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:54 Lat:72 SizeLat:72 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:144 CodeSize:108 Lat:144 SizeLat:144 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:288 CodeSize:216 Lat:288 SizeLat:288 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:62 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:124 Lat:176 SizeLat:176 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:352 CodeSize:248 Lat:352 SizeLat:352 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:36 Lat:40 SizeLat:40 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -470,9 +470,9 @@ define i32 @smul(i32 %arg) {
;
; MVE-LABEL: 'smul'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:912 CodeSize:74 Lat:108 SizeLat:108 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:1824 CodeSize:144 Lat:212 SizeLat:212 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:3648 CodeSize:284 Lat:420 SizeLat:420 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:928 CodeSize:82 Lat:124 SizeLat:124 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1856 CodeSize:160 Lat:244 SizeLat:244 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:3712 CodeSize:316 Lat:484 SizeLat:484 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:284 CodeSize:150 Lat:152 SizeLat:152 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:872 CodeSize:328 Lat:332 SizeLat:332 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -571,9 +571,9 @@ define i32 @umul(i32 %arg) {
;
; MVE-LABEL: 'umul'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:876 CodeSize:38 Lat:72 SizeLat:72 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:1752 CodeSize:72 Lat:140 SizeLat:140 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:3504 CodeSize:140 Lat:276 SizeLat:276 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:892 CodeSize:46 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1784 CodeSize:88 Lat:172 SizeLat:172 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:3568 CodeSize:172 Lat:340 SizeLat:340 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:186 CodeSize:149 Lat:150 SizeLat:150 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:484 CodeSize:326 Lat:328 SizeLat:328 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
index 5d227a8359150..3f92a3836f7ff 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
@@ -34,27 +34,27 @@ declare <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
define i32 @add(i32 %arg) {
; V8M-LABEL: 'add'
; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:30 CodeSize:34 Lat:30 SizeLat:30 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:60 CodeSize:68 Lat:60 SizeLat:60 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:120 CodeSize:136 Lat:120 SizeLat:120 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:256 CodeSize:320 Lat:256 SizeLat:256 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:256 CodeSize:320 Lat:256 SizeLat:256 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:512 CodeSize:640 Lat:512 SizeLat:512 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:576 CodeSize:704 Lat:576 SizeLat:576 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'add'
@@ -84,22 +84,22 @@ define i32 @add(i32 %arg) {
;
; MVE-LABEL: 'add'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:166 CodeSize:112 Lat:166 SizeLat:166 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:298 CodeSize:185 Lat:298 SizeLat:298 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:594 CodeSize:369 Lat:594 SizeLat:594 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:214 CodeSize:136 Lat:214 SizeLat:214 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:394 CodeSize:233 Lat:394 SizeLat:394 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:786 CodeSize:465 Lat:786 SizeLat:786 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -165,27 +165,27 @@ declare <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
define i32 @sub(i32 %arg) {
; V8M-LABEL: 'sub'
; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:30 CodeSize:34 Lat:30 SizeLat:30 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:60 CodeSize:68 Lat:60 SizeLat:60 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:120 CodeSize:136 Lat:120 SizeLat:120 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:256 CodeSize:320 Lat:256 SizeLat:256 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:32 SizeLat:32 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:64 SizeLat:64 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:128 SizeLat:128 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:256 CodeSize:320 Lat:256 SizeLat:256 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:512 CodeSize:640 Lat:512 SizeLat:512 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:576 CodeSize:704 Lat:576 SizeLat:576 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'sub'
@@ -215,22 +215,22 @@ define i32 @sub(i32 %arg) {
;
; MVE-LABEL: 'sub'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:166 CodeSize:112 Lat:166 SizeLat:166 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:298 CodeSize:185 Lat:298 SizeLat:298 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:594 CodeSize:369 Lat:594 SizeLat:594 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:214 CodeSize:136 Lat:214 SizeLat:214 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:394 CodeSize:233 Lat:394 SizeLat:394 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:786 CodeSize:465 Lat:786 SizeLat:786 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:100 CodeSize:70 Lat:100 SizeLat:100 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
index c843331c55e6e..83d1192ecba6a 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
@@ -84,22 +84,22 @@ define i32 @add(i32 %arg) {
;
; MVE-LABEL: 'add'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:55 Lat:74 SizeLat:74 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:110 Lat:148 SizeLat:148 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:296 CodeSize:220 Lat:296 SizeLat:296 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -215,22 +215,22 @@ define i32 @sub(i32 %arg) {
;
; MVE-LABEL: 'sub'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:55 Lat:74 SizeLat:74 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:110 Lat:148 SizeLat:148 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:296 CodeSize:220 Lat:296 SizeLat:296 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/cmps.ll b/llvm/test/Analysis/CostModel/ARM/cmps.ll
index 6b19be542a381..5a648f82611bb 100644
--- a/llvm/test/Analysis/CostModel/ARM/cmps.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cmps.ll
@@ -20,7 +20,7 @@ define i32 @cmps() {
; CHECK-MVE-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a10 = fcmp olt <8 x half> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:24 CodeSize:12 Lat:24 SizeLat:24 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
@@ -30,17 +30,17 @@ define i32 @cmps() {
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 32 for: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 48 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 24 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %g = icmp sge <4 x i32> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 6 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; CHECK-V8M-BASE-LABEL: 'cmps'
@@ -48,17 +48,17 @@ define i32 @cmps() {
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 32 for: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 48 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 24 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %g = icmp sge <4 x i32> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 6 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; CHECK-V8R-LABEL: 'cmps'
@@ -72,9 +72,9 @@ define i32 @cmps() {
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8R-NEXT: Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8R-NEXT: Cost Model: Found costs of 48 for: %a10 = fcmp olt <8 x half> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8R-NEXT: Cost Model: Found costs of 4 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8R-NEXT: Cost Model: Found costs of 10 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
@@ -121,10 +121,10 @@ define void @minmax() {
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %c6 = icmp slt <4 x ptr> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: ret void
;
@@ -136,10 +136,10 @@ define void @minmax() {
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %c6 = icmp slt <4 x ptr> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll b/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
index af548f6216370..a96b040eacbe5 100644
--- a/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
@@ -137,15 +137,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f32u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f32(<2 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:76 CodeSize:5 Lat:9 SizeLat:9 for: %v2f32s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f32(<2 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:72 CodeSize:3 Lat:5 SizeLat:5 for: %v2f32u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f32(<2 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:80 CodeSize:35 Lat:45 SizeLat:45 for: %v2f64s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:96 CodeSize:43 Lat:61 SizeLat:61 for: %v2f64s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:62 CodeSize:22 Lat:27 SizeLat:27 for: %v2f64s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:62 CodeSize:22 Lat:27 SizeLat:27 for: %v2f64s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:62 CodeSize:22 Lat:27 SizeLat:27 for: %v2f64s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:94 CodeSize:22 Lat:27 SizeLat:27 for: %v2f64s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:110 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:84 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:6 CodeSize:7 Lat:6 SizeLat:6 for: %v4f32s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:6 CodeSize:7 Lat:6 SizeLat:6 for: %v4f32u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f32(<4 x float> undef)
@@ -157,15 +157,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4f32u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:284 CodeSize:6 Lat:11 SizeLat:11 for: %v4f32s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:278 CodeSize:3 Lat:5 SizeLat:5 for: %v4f32u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f32(<4 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:234 CodeSize:69 Lat:89 SizeLat:89 for: %v4f64s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:266 CodeSize:85 Lat:121 SizeLat:121 for: %v4f64s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:196 CodeSize:42 Lat:51 SizeLat:51 for: %v4f64s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:196 CodeSize:42 Lat:51 SizeLat:51 for: %v4f64s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:196 CodeSize:42 Lat:51 SizeLat:51 for: %v4f64s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:324 CodeSize:43 Lat:53 SizeLat:53 for: %v4f64s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:356 CodeSize:59 Lat:85 SizeLat:85 for: %v4f64s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:304 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %v8f32s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %v8f32u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f32(<8 x float> undef)
@@ -177,15 +177,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %v8f32u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1148 CodeSize:43 Lat:53 SizeLat:53 for: %v8f32s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1104 CodeSize:5 Lat:9 SizeLat:9 for: %v8f32u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f32(<8 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:762 CodeSize:137 Lat:177 SizeLat:177 for: %v8f64s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:826 CodeSize:169 Lat:241 SizeLat:241 for: %v8f64s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:684 CodeSize:82 Lat:99 SizeLat:99 for: %v8f64s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:114 Lat:163 SizeLat:163 for: %v8f64s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:684 CodeSize:82 Lat:99 SizeLat:99 for: %v8f64s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:114 Lat:163 SizeLat:163 for: %v8f64s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:684 CodeSize:83 Lat:101 SizeLat:101 for: %v8f64s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:115 Lat:165 SizeLat:165 for: %v8f64s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:648 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1192 CodeSize:85 Lat:105 SizeLat:105 for: %v8f64s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1256 CodeSize:117 Lat:169 SizeLat:169 for: %v8f64s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1152 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %v16f32s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %v16f32u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f32(<16 x float> undef)
@@ -197,15 +197,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v16f32u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4488 CodeSize:85 Lat:105 SizeLat:105 for: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4400 CodeSize:9 Lat:17 SizeLat:17 for: %v16f32u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f32(<16 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2698 CodeSize:273 Lat:353 SizeLat:353 for: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2826 CodeSize:337 Lat:481 SizeLat:481 for: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2474 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2540 CodeSize:162 Lat:195 SizeLat:195 for: %v16f64s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2668 CodeSize:226 Lat:323 SizeLat:323 for: %v16f64s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2474 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2540 CodeSize:163 Lat:197 SizeLat:197 for: %v16f64s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2668 CodeSize:227 Lat:325 SizeLat:325 for: %v16f64s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2472 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2536 CodeSize:165 Lat:201 SizeLat:201 for: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2664 CodeSize:229 Lat:329 SizeLat:329 for: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2464 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4560 CodeSize:169 Lat:209 SizeLat:209 for: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4688 CodeSize:233 Lat:337 SizeLat:337 for: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4480 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
index 42563e8e34a94..5b8840c246d99 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
@@ -32,22 +32,22 @@ declare <64 x i8> @llvm.abs.v64i8(<64 x i8>, i1)
define i32 @abs(i32 %arg) {
; MVE-LABEL: 'abs'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.abs.i64(i64 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:55 Lat:74 SizeLat:74 for: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:110 Lat:148 SizeLat:148 for: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:296 CodeSize:220 Lat:296 SizeLat:296 for: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.abs.i32(i32 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.abs.i16(i16 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.abs.i8(i8 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:29 Lat:40 SizeLat:40 for: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
index 4e6ebbc593961..6e80420152588 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
@@ -6,24 +6,24 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define void @icmp() {
; CHECK-LABEL: 'icmp'
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:10 Lat:20 SizeLat:20 for: %v2i8 = icmp slt <2 x i8> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i8 = icmp slt <2 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i8 = icmp slt <4 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v8i8 = icmp slt <8 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v16i8 = icmp slt <16 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:132 CodeSize:130 Lat:132 SizeLat:132 for: %v32i8 = icmp slt <32 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:10 Lat:20 SizeLat:20 for: %v2i16 = icmp slt <2 x i16> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i16 = icmp slt <2 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i16 = icmp slt <4 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v8i16 = icmp slt <8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:68 CodeSize:66 Lat:68 SizeLat:68 for: %v16i16 = icmp slt <16 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:10 Lat:20 SizeLat:20 for: %v2i32 = icmp slt <2 x i32> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i32 = icmp slt <2 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i32 = icmp slt <4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:34 Lat:36 SizeLat:36 for: %v8i32 = icmp slt <8 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:72 CodeSize:68 Lat:72 SizeLat:72 for: %v16i32 = icmp slt <16 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i64 = icmp slt <2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:72 CodeSize:36 Lat:72 SizeLat:72 for: %v4i64 = icmp slt <4 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:144 CodeSize:72 Lat:144 SizeLat:144 for: %v8i64 = icmp slt <8 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:68 CodeSize:34 Lat:68 SizeLat:68 for: %v2i128 = icmp slt <2 x i128> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:136 CodeSize:68 Lat:136 SizeLat:136 for: %v4i128 = icmp slt <4 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:52 CodeSize:26 Lat:52 SizeLat:52 for: %v2i64 = icmp slt <2 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:104 CodeSize:52 Lat:104 SizeLat:104 for: %v4i64 = icmp slt <4 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:208 CodeSize:104 Lat:208 SizeLat:208 for: %v8i64 = icmp slt <8 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:84 CodeSize:42 Lat:84 SizeLat:84 for: %v2i128 = icmp slt <2 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:168 CodeSize:84 Lat:168 SizeLat:168 for: %v4i128 = icmp slt <4 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2i8 = icmp slt <2 x i8> undef, undef
@@ -76,9 +76,9 @@ define void @fcmp() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4f32 = fcmp olt <4 x float> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:36 CodeSize:34 Lat:36 SizeLat:36 for: %v8f32 = fcmp olt <8 x float> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:72 CodeSize:68 Lat:72 SizeLat:72 for: %v16f32 = fcmp olt <16 x float> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v2f64 = fcmp olt <2 x double> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %v4f64 = fcmp olt <4 x double> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:32 CodeSize:16 Lat:32 SizeLat:32 for: %v8f64 = fcmp olt <8 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:12 Lat:24 SizeLat:24 for: %v2f64 = fcmp olt <2 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:48 CodeSize:24 Lat:48 SizeLat:48 for: %v4f64 = fcmp olt <4 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:96 CodeSize:48 Lat:96 SizeLat:96 for: %v8f64 = fcmp olt <8 x double> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f16 = fcmp olt <2 x half> undef, undef
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
index e4cc8fed5052c..af6c694bb42f9 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
@@ -33,22 +33,22 @@ declare <64 x i8> @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
define i32 @smin(i32 %arg) {
; MVE-LABEL: 'smin'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:76 CodeSize:38 Lat:76 SizeLat:76 for: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:152 CodeSize:76 Lat:152 SizeLat:152 for: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -115,22 +115,22 @@ declare <64 x i8> @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
define i32 @smax(i32 %arg) {
; MVE-LABEL: 'smax'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:76 CodeSize:38 Lat:76 SizeLat:76 for: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:152 CodeSize:76 Lat:152 SizeLat:152 for: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -198,22 +198,22 @@ declare <64 x i8> @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
define i32 @umin(i32 %arg) {
; MVE-LABEL: 'umin'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:76 CodeSize:38 Lat:76 SizeLat:76 for: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:152 CodeSize:76 Lat:152 SizeLat:152 for: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -280,22 +280,22 @@ declare <64 x i8> @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
define i32 @umax(i32 %arg) {
; MVE-LABEL: 'umax'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:76 CodeSize:38 Lat:76 SizeLat:76 for: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:152 CodeSize:76 Lat:152 SizeLat:152 for: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:22 CodeSize:11 Lat:22 SizeLat:22 for: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
index 861d825b3ecdb..98d07ac71b42e 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:13 CodeSize:14 Lat:13 SizeLat:13 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:35 CodeSize:38 Lat:35 SizeLat:35 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:79 CodeSize:86 Lat:79 SizeLat:79 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:167 CodeSize:182 Lat:167 SizeLat:167 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:110 CodeSize:59 Lat:110 SizeLat:110 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:212 CodeSize:110 Lat:212 SizeLat:212 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:416 CodeSize:212 Lat:416 SizeLat:416 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:824 CodeSize:416 Lat:824 SizeLat:824 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:763 CodeSize:890 Lat:763 SizeLat:763 for: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
index d8fcc7bca9155..db8ca68723414 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:13 CodeSize:14 Lat:13 SizeLat:13 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:35 CodeSize:38 Lat:35 SizeLat:35 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:79 CodeSize:86 Lat:79 SizeLat:79 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:167 CodeSize:182 Lat:167 SizeLat:167 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:110 CodeSize:59 Lat:110 SizeLat:110 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:212 CodeSize:110 Lat:212 SizeLat:212 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:416 CodeSize:212 Lat:416 SizeLat:416 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:824 CodeSize:416 Lat:824 SizeLat:824 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:763 CodeSize:890 Lat:763 SizeLat:763 for: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
index b95808a1bb4b5..affea95302336 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:13 CodeSize:14 Lat:13 SizeLat:13 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:35 CodeSize:38 Lat:35 SizeLat:35 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:79 CodeSize:86 Lat:79 SizeLat:79 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:167 CodeSize:182 Lat:167 SizeLat:167 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:110 CodeSize:59 Lat:110 SizeLat:110 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:212 CodeSize:110 Lat:212 SizeLat:212 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:416 CodeSize:212 Lat:416 SizeLat:416 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:824 CodeSize:416 Lat:824 SizeLat:824 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:763 CodeSize:890 Lat:763 SizeLat:763 for: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
index 055606b0f8617..6f3f025aef7a7 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:13 CodeSize:14 Lat:13 SizeLat:13 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:35 CodeSize:38 Lat:35 SizeLat:35 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:79 CodeSize:86 Lat:79 SizeLat:79 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:167 CodeSize:182 Lat:167 SizeLat:167 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:110 CodeSize:59 Lat:110 SizeLat:110 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:212 CodeSize:110 Lat:212 SizeLat:212 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:416 CodeSize:212 Lat:416 SizeLat:416 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:824 CodeSize:416 Lat:824 SizeLat:824 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:7 SizeLat:7 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:19 CodeSize:22 Lat:19 SizeLat:19 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:43 CodeSize:50 Lat:43 SizeLat:43 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:91 CodeSize:106 Lat:91 SizeLat:91 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:187 CodeSize:218 Lat:187 SizeLat:187 for: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:379 CodeSize:442 Lat:379 SizeLat:379 for: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:763 CodeSize:890 Lat:763 SizeLat:763 for: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:58 CodeSize:31 Lat:58 SizeLat:58 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll b/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
index ba53e0f2a90a2..80abbdd1cd90b 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
@@ -6,7 +6,7 @@ define void @exts() {
; VSX is disabled, so this cost needs to include scalarization (because
; <4 x double> is legalized to scalars).
- ; CHECK: cost of 44 {{.*}} fcmp
+ ; CHECK: cost of 56 {{.*}} fcmp
%v1 = fcmp ugt <4 x double> undef, undef
ret void
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast-sat.ll b/llvm/test/Analysis/CostModel/RISCV/cast-sat.ll
index ee70811bf9bd0..bf969e50ff2d8 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast-sat.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast-sat.ll
@@ -7,45 +7,45 @@
define void @fptoui_sat() {
; RV64ZVE32F-LABEL: 'fptoui_sat'
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v1f32_v1i8 = call <1 x i8> @llvm.fptoui.sat.v1i8.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i8 = call <1 x i8> @llvm.fptoui.sat.v1i8.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i8 = call <1 x i8> @llvm.fptoui.sat.v1i8.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i16 = call <1 x i16> @llvm.fptoui.sat.v1i16.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i16 = call <1 x i16> @llvm.fptoui.sat.v1i16.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i16 = call <1 x i16> @llvm.fptoui.sat.v1i16.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i32 = call <1 x i32> @llvm.fptoui.sat.v1i32.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i32 = call <1 x i32> @llvm.fptoui.sat.v1i32.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i32 = call <1 x i32> @llvm.fptoui.sat.v1i32.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i64 = call <1 x i64> @llvm.fptoui.sat.v1i64.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v1f64_v1i64 = call <1 x i64> @llvm.fptoui.sat.v1i64.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v1f64_v1i64 = call <1 x i64> @llvm.fptoui.sat.v1i64.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v1f32_v1i1 = call <1 x i1> @llvm.fptoui.sat.v1i1.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v1f64_v1i1 = call <1 x i1> @llvm.fptoui.sat.v1i1.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v1f64_v1i1 = call <1 x i1> @llvm.fptoui.sat.v1i1.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f32_v2i8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2f32_v2i16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2f32_v2i32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f32_v2i64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f64_v2i64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v2f32_v2i1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v2f64_v2i1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v2f64_v2i1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4f32_v4i8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f32_v4i16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f32_v4i32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v4f32_v4i64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v4f64_v4i64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v4f64_v4i64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v4f32_v4i1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %v4f64_v4i1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %v4f64_v4i1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f32_v8i8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v8f64_v8i8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %v8f64_v8i8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8f32_v8i16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v8f64_v8i16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %v8f64_v8i16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f32_v8i32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %v8f64_v8i32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %v8f64_v8i32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v8f32_v8i64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8f64_v8i64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v8f64_v8i64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v8f32_v8i1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %v8f64_v8i1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f64_v8i1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.fptoui.sat.nxv1i8.nxv1f32(<vscale x 1 x float> poison)
; RV64ZVE32F-NEXT: Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.fptoui.sat.nxv1i8.nxv1f64(<vscale x 1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.fptoui.sat.nxv1i16.nxv1f32(<vscale x 1 x float> poison)
@@ -296,45 +296,45 @@ define void @fptoui_sat() {
define void @fptosi_sat() {
; RV64ZVE32F-LABEL: 'fptosi_sat'
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v1f32_v1i8 = call <1 x i8> @llvm.fptosi.sat.v1i8.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i8 = call <1 x i8> @llvm.fptosi.sat.v1i8.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i8 = call <1 x i8> @llvm.fptosi.sat.v1i8.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i16 = call <1 x i16> @llvm.fptosi.sat.v1i16.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i16 = call <1 x i16> @llvm.fptosi.sat.v1i16.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i16 = call <1 x i16> @llvm.fptosi.sat.v1i16.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i32 = call <1 x i32> @llvm.fptosi.sat.v1i32.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f64_v1i32 = call <1 x i32> @llvm.fptosi.sat.v1i32.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v1f64_v1i32 = call <1 x i32> @llvm.fptosi.sat.v1i32.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v1f32_v1i64 = call <1 x i64> @llvm.fptosi.sat.v1i64.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v1f64_v1i64 = call <1 x i64> @llvm.fptosi.sat.v1i64.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v1f64_v1i64 = call <1 x i64> @llvm.fptosi.sat.v1i64.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v1f32_v1i1 = call <1 x i1> @llvm.fptosi.sat.v1i1.v1f32(<1 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v1f64_v1i1 = call <1 x i1> @llvm.fptosi.sat.v1i1.v1f64(<1 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v1f64_v1i1 = call <1 x i1> @llvm.fptosi.sat.v1i1.v1f64(<1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f32_v2i8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2f32_v2i16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v2f32_v2i32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f64_v2i32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f32_v2i64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f64_v2i64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v2f64_v2i64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v2f32_v2i1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f32(<2 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v2f64_v2i1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v2f64_v2i1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v4f32_v4i8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f32_v4i16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f32_v4i32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v4f64_v4i32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f64_v4i32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v4f32_v4i64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v4f64_v4i64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v4f64_v4i64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v4f32_v4i1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f32(<4 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %v4f64_v4i1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %v4f64_v4i1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f32_v8i8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v8f64_v8i8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %v8f64_v8i8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8f32_v8i16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v8f64_v8i16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %v8f64_v8i16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f32_v8i32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %v8f64_v8i32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %v8f64_v8i32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v8f32_v8i64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8f64_v8i64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v8f64_v8i64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v8f32_v8i1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f32(<8 x float> poison)
-; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %v8f64_v8i1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> poison)
+; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f64_v8i1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.fptosi.sat.nxv1i8.nxv1f32(<vscale x 1 x float> poison)
; RV64ZVE32F-NEXT: Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.fptosi.sat.nxv1i8.nxv1f64(<vscale x 1 x double> poison)
; RV64ZVE32F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.fptosi.sat.nxv1i16.nxv1f32(<vscale x 1 x float> poison)
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-fcmp-f16.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-fcmp-f16.ll
index 8396e80ca3e80..195c67f6c16b4 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-fcmp-f16.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-fcmp-f16.ll
@@ -5,10 +5,10 @@
define void @fcmp_oeq() {
; NOF16-LABEL: 'fcmp_oeq'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp oeq <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp oeq <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp oeq <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp oeq <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp oeq <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp oeq <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp oeq <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp oeq <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp oeq <vscale x 4 x half> undef, undef
@@ -53,10 +53,10 @@ define void @fcmp_oeq() {
}
define void @fcmp_one() {
; NOF16-LABEL: 'fcmp_one'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp one <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp one <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp one <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp one <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp one <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp one <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp one <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp one <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp one <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp one <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp one <vscale x 4 x half> undef, undef
@@ -101,10 +101,10 @@ define void @fcmp_one() {
}
define void @fcmp_olt() {
; NOF16-LABEL: 'fcmp_olt'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp olt <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp olt <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp olt <vscale x 4 x half> undef, undef
@@ -149,10 +149,10 @@ define void @fcmp_olt() {
}
define void @fcmp_ole() {
; NOF16-LABEL: 'fcmp_ole'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ole <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ole <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ole <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ole <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ole <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ole <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ole <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ole <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ole <vscale x 4 x half> undef, undef
@@ -197,10 +197,10 @@ define void @fcmp_ole() {
}
define void @fcmp_ogt() {
; NOF16-LABEL: 'fcmp_ogt'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ogt <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ogt <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ogt <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ogt <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ogt <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ogt <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ogt <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ogt <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ogt <vscale x 4 x half> undef, undef
@@ -245,10 +245,10 @@ define void @fcmp_ogt() {
}
define void @fcmp_oge() {
; NOF16-LABEL: 'fcmp_oge'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp oge <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp oge <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp oge <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp oge <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp oge <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp oge <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp oge <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp oge <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp oge <vscale x 4 x half> undef, undef
@@ -293,10 +293,10 @@ define void @fcmp_oge() {
}
define void @fcmp_ueq() {
; NOF16-LABEL: 'fcmp_ueq'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ueq <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ueq <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ueq <vscale x 4 x half> undef, undef
@@ -341,10 +341,10 @@ define void @fcmp_ueq() {
}
define void @fcmp_une() {
; NOF16-LABEL: 'fcmp_une'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp une <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp une <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp une <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp une <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp une <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp une <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp une <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp une <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp une <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp une <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp une <vscale x 4 x half> undef, undef
@@ -389,10 +389,10 @@ define void @fcmp_une() {
}
define void @fcmp_ult() {
; NOF16-LABEL: 'fcmp_ult'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ult <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ult <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ult <vscale x 4 x half> undef, undef
@@ -437,10 +437,10 @@ define void @fcmp_ult() {
}
define void @fcmp_ule() {
; NOF16-LABEL: 'fcmp_ule'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ule <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ule <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ule <vscale x 4 x half> undef, undef
@@ -485,10 +485,10 @@ define void @fcmp_ule() {
}
define void @fcmp_ugt() {
; NOF16-LABEL: 'fcmp_ugt'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp ugt <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp ugt <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp ugt <vscale x 4 x half> undef, undef
@@ -533,10 +533,10 @@ define void @fcmp_ugt() {
}
define void @fcmp_uge() {
; NOF16-LABEL: 'fcmp_uge'
-; NOF16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef
-; NOF16-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef
+; NOF16-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv1f16 = fcmp uge <vscale x 1 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv2f16 = fcmp uge <vscale x 2 x half> undef, undef
; NOF16-NEXT: Cost Model: Invalid cost for instruction: %nxv4f16 = fcmp uge <vscale x 4 x half> undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
index 41bf88b1ec316..f19cdc759b495 100644
--- a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
+++ b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
@@ -857,45 +857,45 @@ define void @fp16() {
; SSE2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u32 = call i32 @llvm.fptoui.sat.i32.f16(half undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16s64 = call i64 @llvm.fptosi.sat.i64.f16(half undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u64 = call i64 @llvm.fptoui.sat.i64.f16(half undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f16(<2 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v2f16u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f16(<2 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v2f16u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f16(<2 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f16(<2 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f16(<2 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f16u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v4f16u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 99 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 104 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 79 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 95 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 103 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 92 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 94 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 197 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 185 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 216 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 190 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 221 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 184 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 215 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 188 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 219 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -910,45 +910,45 @@ define void @fp16() {
; SSE42-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u32 = call i32 @llvm.fptoui.sat.i32.f16(half undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16s64 = call i64 @llvm.fptosi.sat.i64.f16(half undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u64 = call i64 @llvm.fptoui.sat.i64.f16(half undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f16(<2 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v2f16u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f16(<2 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v2f16u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f16(<2 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f16(<2 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f16(<2 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 35 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %v4f16u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 35 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %v4f16u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 99 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 94 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 77 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 95 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 103 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 92 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 94 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 197 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 213 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 185 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 201 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 190 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 206 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 184 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 200 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 188 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 204 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -963,45 +963,45 @@ define void @fp16() {
; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u32 = call i32 @llvm.fptoui.sat.i32.f16(half undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16s64 = call i64 @llvm.fptosi.sat.i64.f16(half undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u64 = call i64 @llvm.fptoui.sat.i64.f16(half undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f16(<2 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v2f16u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f16(<2 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v2f16u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f16(<2 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f16(<2 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f16(<2 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v4f16u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v4f16u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 76 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 84 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 59 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 105 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 113 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 103 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 111 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 86 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 101 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 84 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 92 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 203 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 219 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 170 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 212 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 179 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 212 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 179 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 210 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 226 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 175 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 208 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -1122,45 +1122,45 @@ define void @fp16() {
; SLM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u32 = call i32 @llvm.fptoui.sat.i32.f16(half undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16s64 = call i64 @llvm.fptosi.sat.i64.f16(half undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f16u64 = call i64 @llvm.fptoui.sat.i64.f16(half undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v2f16s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f16(<2 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f16(<2 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %v2f16s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f16(<2 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %v2f16u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f16(<2 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v2f16s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f16(<2 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v2f16u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f16(<2 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f16(<2 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f16(<2 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %v2f16s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f16(<2 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v2f16u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f16(<2 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %v4f16s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %v4f16u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %v4f16s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %v4f16u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 101 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 104 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 77 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 97 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 105 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 92 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 110 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 199 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 215 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 187 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 203 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 194 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 210 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 184 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 200 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 204 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 220 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
index e3273b563192f..dd7352ef2b26b 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
@@ -27,7 +27,7 @@ define void @expensive_icmp(ptr noalias nocapture %d, ptr nocapture readonly %s,
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK: Cost of 18 for VF 2: WIDEN ir<%1> = load vp<[[VP5]]>
; CHECK: Cost of 4 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%1> to i32
-; CHECK: Cost of 20 for VF 2: WIDEN ir<%cmp2> = icmp sgt ir<%conv>, ir<%conv1>
+; CHECK: Cost of 36 for VF 2: WIDEN ir<%cmp2> = icmp sgt ir<%conv>, ir<%conv1>
; CHECK: Cost of 26 for VF 2: WIDEN ir<%conv6> = add ir<%1>, ir<%0>
; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx7> = getelementptr ir<%d>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%arrayidx7>, ir<1>
@@ -45,7 +45,7 @@ define void @expensive_icmp(ptr noalias nocapture %d, ptr nocapture readonly %s,
; CHECK: Cost of 0 for VF 2: IR %cmp2 = icmp sgt i32 %conv, %conv1
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost for VF 2: 86 (Estimated cost per lane: 43)
+; CHECK: Cost for VF 2: 102 (Estimated cost per lane: 51)
; CHECK: Cost of 1 for VF 4: induction instruction %inc = add nuw nsw i32 %i.016, 1
; CHECK: Cost of 0 for VF 4: induction instruction %i.016 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
@@ -179,7 +179,7 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 26 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv3>, ir<%conv1>
; CHECK: Cost of 18 for VF 2: WIDEN ir<%shr> = ashr ir<%mul>, ir<7>
; CHECK: Cost of 0 for VF 2: WIDEN ir<%2> = icmp slt ir<%shr>, ir<127>
-; CHECK: Cost of 22 for VF 2: WIDEN ir<%spec.select.i> = select ir<%2>, ir<%shr>, ir<127>
+; CHECK: Cost of 38 for VF 2: WIDEN ir<%spec.select.i> = select ir<%2>, ir<%shr>, ir<127>
; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%conv4> = trunc ir<%spec.select.i> to i8
; CHECK: Cost of 0 for VF 2: vp<[[VP13:%[0-9]+]]> = vector-pointer vp<%next.gep>.1, ir<1>
; CHECK: Cost of 18 for VF 2: WIDEN store vp<[[VP13]]>, ir<%conv4>
@@ -215,7 +215,7 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 0 for VF 2: vp<[[VP6]]> = DERIVED-IV ir<%pSrcB> + vp<[[VP2]]> * ir<1>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost for VF 2: 130 (Estimated cost per lane: 65)
+; CHECK: Cost for VF 2: 146 (Estimated cost per lane: 73)
; CHECK: Cost of 1 for VF 4: induction instruction %dec = add i32 %blkCnt.012, -1
; CHECK: Cost of 0 for VF 4: induction instruction %blkCnt.012 = phi i32 [ %dec, %while.body ], [ %blockSize, %while.body.preheader ]
; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index 57ca3db075689..8947279e9b052 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -357,17 +357,20 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
; GFX8-NEXT: bb:
; GFX8-NEXT: [[ARG0_0:%.*]] = extractelement <4 x i16> [[ARG0:%.*]], i64 0
; GFX8-NEXT: [[ARG0_1:%.*]] = extractelement <4 x i16> [[ARG0]], i64 1
+; GFX8-NEXT: [[ARG0_2:%.*]] = extractelement <4 x i16> [[ARG0]], i64 2
+; GFX8-NEXT: [[ARG0_3:%.*]] = extractelement <4 x i16> [[ARG0]], i64 3
; GFX8-NEXT: [[ARG1_0:%.*]] = extractelement <4 x i16> [[ARG1:%.*]], i64 0
; GFX8-NEXT: [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
+; GFX8-NEXT: [[ARG1_2:%.*]] = extractelement <4 x i16> [[ARG1]], i64 2
+; GFX8-NEXT: [[ARG1_3:%.*]] = extractelement <4 x i16> [[ARG1]], i64 3
; GFX8-NEXT: [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
; GFX8-NEXT: [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP3]])
+; GFX8-NEXT: [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT: [[ADD_3:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
; GFX8-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
; GFX8-NEXT: [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
-; GFX8-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; GFX8-NEXT: [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; GFX8-NEXT: [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX8-NEXT: [[INS_31:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
; GFX8-NEXT: ret <4 x i16> [[INS_31]]
;
; GFX9-LABEL: @uadd_sat_v4i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll b/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
index 2370505610f6c..65b7a2643c252 100644
--- a/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
+++ b/llvm/test/Transforms/SLPVectorizer/scalarazied-result.ll
@@ -5,6 +5,10 @@
define void @test() {
; X86-LABEL: @test(
; X86-NEXT: entry:
+; X86-NEXT: [[TMP0:%.*]] = extractelement <8 x half> zeroinitializer, i64 1
+; X86-NEXT: [[TOBOOL:%.*]] = fcmp une half [[TMP0]], 0.000000e+00
+; X86-NEXT: [[TMP1:%.*]] = extractelement <8 x half> zeroinitializer, i64 1
+; X86-NEXT: [[TOBOOL3:%.*]] = fcmp une half [[TMP1]], 0.000000e+00
; X86-NEXT: ret void
;
; AARCH64-LABEL: @test(
>From 6f89c0f78af3ddcd80949d50749db4394d4fd8c1 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 7 Jul 2026 13:31:01 +0800
Subject: [PATCH 3/7] Fix fcmp-zve64x.ll test
---
llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll b/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
index b98faa5bacea6..52feae55c8450 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fcmp-zve64x.ll
@@ -3,8 +3,8 @@
define void @f() {
; CHECK-LABEL: 'f'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = fcmp oge <4 x float> poison, poison
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = fcmp oge <4 x double> poison, poison
+; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %1 = fcmp oge <4 x float> poison, poison
+; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %2 = fcmp oge <4 x double> poison, poison
; CHECK-NEXT: Cost Model: Invalid cost for instruction: %3 = fcmp oge <vscale x 2 x float> poison, poison
; CHECK-NEXT: Cost Model: Invalid cost for instruction: %4 = fcmp oge <vscale x 2 x double> poison, poison
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
>From 656d9cc16777cd180e5432e9c1fedce9dae610e5 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 13 Jul 2026 19:19:40 +0800
Subject: [PATCH 4/7] Avoid changes to select
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 5be9e21b1d6dd..3b5164611f9ba 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1464,13 +1464,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// Return the cost of multiple scalar invocation plus the cost of
// inserting and extracting the values.
- InstructionCost Overhead =
- getScalarizationOverhead(ValVTy, /*Insert*/ false,
- /*Extract*/ true, CostKind);
+ InstructionCost Overhead;
if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
- Overhead +=
+ Overhead =
+ getScalarizationOverhead(ValVTy, /*Insert*/ false,
+ /*Extract*/ true, CostKind) +
getScalarizationOverhead(cast<VectorType>(CondTy), /*Insert*/ true,
/*Extract*/ false, CostKind);
+ else
+ Overhead = getScalarizationOverhead(ValVTy, /*Insert*/ true,
+ /*Extract*/ false, CostKind);
return Overhead + Num * Cost;
}
>From 4c536f9777cc9974c2e4bcf8a2f3a1fb967f50ee Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 13 Jul 2026 19:21:31 +0800
Subject: [PATCH 5/7] Use getOperandsScalarizationOverhead
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3b5164611f9ba..f538a0c965b1a 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1467,8 +1467,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
InstructionCost Overhead;
if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
Overhead =
- getScalarizationOverhead(ValVTy, /*Insert*/ false,
- /*Extract*/ true, CostKind) +
+ getOperandsScalarizationOverhead({ValTy, ValTy}, CostKind) +
getScalarizationOverhead(cast<VectorType>(CondTy), /*Insert*/ true,
/*Extract*/ false, CostKind);
else
>From 206ed794979200752d07563e1b16228e3ad4b676 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 13 Jul 2026 19:24:41 +0800
Subject: [PATCH 6/7] Update tests w/ getOperandsScalarizationOverhead change
---
llvm/test/Analysis/CostModel/AArch64/fshl.ll | 2 +-
llvm/test/Analysis/CostModel/AArch64/fshr.ll | 2 +-
.../CostModel/ARM/active_lane_mask.ll | 24 ++---
.../Analysis/CostModel/ARM/arith-overflow.ll | 36 +++----
.../test/Analysis/CostModel/ARM/arith-ssat.ll | 96 +++++++++----------
.../test/Analysis/CostModel/ARM/arith-usat.ll | 24 ++---
llvm/test/Analysis/CostModel/ARM/cmps.ll | 42 ++++----
llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll | 40 ++++----
llvm/test/Analysis/CostModel/ARM/mve-abs.ll | 12 +--
llvm/test/Analysis/CostModel/ARM/mve-cmp.ll | 22 ++---
.../test/Analysis/CostModel/ARM/mve-minmax.ll | 48 +++++-----
.../Analysis/CostModel/ARM/reduce-smax.ll | 58 +++++------
.../Analysis/CostModel/ARM/reduce-smin.ll | 58 +++++------
.../Analysis/CostModel/ARM/reduce-umax.ll | 58 +++++------
.../Analysis/CostModel/ARM/reduce-umin.ll | 58 +++++------
.../CostModel/PowerPC/cmp-expanded.ll | 2 +-
llvm/test/Analysis/CostModel/X86/fptoi_sat.ll | 80 ++++++++--------
.../LoopVectorize/ARM/mve-icmpcost.ll | 8 +-
18 files changed, 335 insertions(+), 335 deletions(-)
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshl.ll b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
index 2264a77acdd92..fe6cc1860ac95 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshl.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
@@ -286,7 +286,7 @@ entry:
define <2 x i128> @fshl_v2i128_3rd_arg_var(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-LABEL: 'fshl_v2i128_3rd_arg_var'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:19 Lat:25 SizeLat:25 for: %r = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
+; CHECK-NEXT: Cost Model: Found costs of RThru:44 CodeSize:21 Lat:29 SizeLat:29 for: %r = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %r
;
entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshr.ll b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
index 6c8c472268c49..9fe06aaf48d9c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshr.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
@@ -286,7 +286,7 @@ entry:
define <2 x i128> @fshr_v2i128_3rd_arg_var(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-LABEL: 'fshr_v2i128_3rd_arg_var'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:19 Lat:25 SizeLat:25 for: %r = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
+; CHECK-NEXT: Cost Model: Found costs of RThru:44 CodeSize:21 Lat:29 SizeLat:29 for: %r = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %r
;
entry:
diff --git a/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll b/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
index e0cf7ce9a7ea4..90faeb1dbb105 100644
--- a/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
+++ b/llvm/test/Analysis/CostModel/ARM/active_lane_mask.ll
@@ -3,18 +3,18 @@
define void @get_lane_mask() {
; CHECK-LABEL: 'get_lane_mask'
-; CHECK-NEXT: Cost Model: Found costs of RThru:192 CodeSize:208 Lat:192 SizeLat:192 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:96 CodeSize:104 Lat:96 SizeLat:96 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:48 CodeSize:52 Lat:48 SizeLat:48 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:24 CodeSize:26 Lat:24 SizeLat:24 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:112 CodeSize:128 Lat:112 SizeLat:112 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:56 CodeSize:64 Lat:56 SizeLat:56 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:28 CodeSize:32 Lat:28 SizeLat:28 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:16 Lat:14 SizeLat:14 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:112 CodeSize:128 Lat:112 SizeLat:112 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:56 CodeSize:64 Lat:56 SizeLat:56 for: %mask_v8i1_i16 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:28 CodeSize:32 Lat:28 SizeLat:28 for: %mask_v4i1_i16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i16(i16 undef, i16 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:16 Lat:14 SizeLat:14 for: %mask_v2i1_i16 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:224 CodeSize:240 Lat:224 SizeLat:224 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:112 CodeSize:120 Lat:112 SizeLat:112 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:56 CodeSize:60 Lat:56 SizeLat:56 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:28 CodeSize:30 Lat:28 SizeLat:28 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:18 Lat:16 SizeLat:16 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %mask_v8i1_i16 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %mask_v4i1_i16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i16(i16 undef, i16 undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:18 Lat:16 SizeLat:16 for: %mask_v2i1_i16 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i16(i16 undef, i16 undef)
; CHECK-NEXT: Cost Model: Found costs of 1 for: ret void
;
%mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
index cd683d0c9cb43..796b8bf162c3d 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-overflow.ll
@@ -66,9 +66,9 @@ define i32 @sadd(i32 %arg) {
;
; MVE-LABEL: 'sadd'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:108 Lat:158 SizeLat:158 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:282 CodeSize:177 Lat:282 SizeLat:282 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:562 CodeSize:353 Lat:562 SizeLat:562 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:222 CodeSize:140 Lat:222 SizeLat:222 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:410 CodeSize:241 Lat:410 SizeLat:410 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:818 CodeSize:481 Lat:818 SizeLat:818 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:78 CodeSize:71 Lat:78 SizeLat:78 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -167,9 +167,9 @@ define i32 @uadd(i32 %arg) {
;
; MVE-LABEL: 'uadd'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:62 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:124 Lat:176 SizeLat:176 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:352 CodeSize:248 Lat:352 SizeLat:352 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:120 CodeSize:78 Lat:120 SizeLat:120 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:240 CodeSize:156 Lat:240 SizeLat:240 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:312 Lat:480 SizeLat:480 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:36 Lat:40 SizeLat:40 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -268,9 +268,9 @@ define i32 @ssub(i32 %arg) {
;
; MVE-LABEL: 'ssub'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:108 Lat:158 SizeLat:158 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:282 CodeSize:177 Lat:282 SizeLat:282 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:562 CodeSize:353 Lat:562 SizeLat:562 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:222 CodeSize:140 Lat:222 SizeLat:222 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:410 CodeSize:241 Lat:410 SizeLat:410 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:818 CodeSize:481 Lat:818 SizeLat:818 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:78 CodeSize:71 Lat:78 SizeLat:78 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -369,9 +369,9 @@ define i32 @usub(i32 %arg) {
;
; MVE-LABEL: 'usub'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:62 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:124 Lat:176 SizeLat:176 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:352 CodeSize:248 Lat:352 SizeLat:352 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:120 CodeSize:78 Lat:120 SizeLat:120 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:240 CodeSize:156 Lat:240 SizeLat:240 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:312 Lat:480 SizeLat:480 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:40 CodeSize:36 Lat:40 SizeLat:40 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -470,9 +470,9 @@ define i32 @smul(i32 %arg) {
;
; MVE-LABEL: 'smul'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:928 CodeSize:82 Lat:124 SizeLat:124 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:1856 CodeSize:160 Lat:244 SizeLat:244 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:3712 CodeSize:316 Lat:484 SizeLat:484 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:960 CodeSize:98 Lat:156 SizeLat:156 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1920 CodeSize:192 Lat:308 SizeLat:308 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:3840 CodeSize:380 Lat:612 SizeLat:612 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:284 CodeSize:150 Lat:152 SizeLat:152 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:872 CodeSize:328 Lat:332 SizeLat:332 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -571,9 +571,9 @@ define i32 @umul(i32 %arg) {
;
; MVE-LABEL: 'umul'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:892 CodeSize:46 Lat:88 SizeLat:88 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:1784 CodeSize:88 Lat:172 SizeLat:172 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:3568 CodeSize:172 Lat:340 SizeLat:340 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:924 CodeSize:62 Lat:120 SizeLat:120 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1848 CodeSize:120 Lat:236 SizeLat:236 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:3696 CodeSize:236 Lat:468 SizeLat:468 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
; MVE-NEXT: Cost Model: Found costs of RThru:186 CodeSize:149 Lat:150 SizeLat:150 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:484 CodeSize:326 Lat:328 SizeLat:328 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
index 3f92a3836f7ff..0397fd4d088a2 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-ssat.ll
@@ -34,27 +34,27 @@ declare <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
define i32 @add(i32 %arg) {
; V8M-LABEL: 'add'
; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:40 Lat:36 SizeLat:36 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:80 Lat:72 SizeLat:72 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:160 Lat:144 SizeLat:144 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:320 CodeSize:384 Lat:320 SizeLat:320 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:576 CodeSize:704 Lat:576 SizeLat:576 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:320 CodeSize:384 Lat:320 SizeLat:320 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:640 CodeSize:768 Lat:640 SizeLat:640 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'add'
@@ -84,22 +84,22 @@ define i32 @add(i32 %arg) {
;
; MVE-LABEL: 'add'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:214 CodeSize:136 Lat:214 SizeLat:214 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:394 CodeSize:233 Lat:394 SizeLat:394 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:786 CodeSize:465 Lat:786 SizeLat:786 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:310 CodeSize:184 Lat:310 SizeLat:310 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:586 CodeSize:329 Lat:586 SizeLat:586 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1170 CodeSize:657 Lat:1170 SizeLat:1170 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -165,27 +165,27 @@ declare <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
define i32 @sub(i32 %arg) {
; V8M-LABEL: 'sub'
; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:32 CodeSize:36 Lat:32 SizeLat:32 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:64 CodeSize:72 Lat:64 SizeLat:64 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:128 CodeSize:144 Lat:128 SizeLat:128 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:40 Lat:36 SizeLat:36 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:80 Lat:72 SizeLat:72 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:160 Lat:144 SizeLat:144 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:320 CodeSize:384 Lat:320 SizeLat:320 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:18 CodeSize:22 Lat:18 SizeLat:18 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:36 CodeSize:44 Lat:36 SizeLat:36 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:72 CodeSize:88 Lat:72 SizeLat:72 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:144 CodeSize:176 Lat:144 SizeLat:144 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:288 CodeSize:352 Lat:288 SizeLat:288 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:576 CodeSize:704 Lat:576 SizeLat:576 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:40 SizeLat:40 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:80 SizeLat:80 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:160 SizeLat:160 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:320 CodeSize:384 Lat:320 SizeLat:320 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:640 CodeSize:768 Lat:640 SizeLat:640 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'sub'
@@ -215,22 +215,22 @@ define i32 @sub(i32 %arg) {
;
; MVE-LABEL: 'sub'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:214 CodeSize:136 Lat:214 SizeLat:214 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:394 CodeSize:233 Lat:394 SizeLat:394 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:786 CodeSize:465 Lat:786 SizeLat:786 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:310 CodeSize:184 Lat:310 SizeLat:310 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:586 CodeSize:329 Lat:586 SizeLat:586 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1170 CodeSize:657 Lat:1170 SizeLat:1170 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:148 CodeSize:94 Lat:148 SizeLat:148 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:196 CodeSize:118 Lat:196 SizeLat:196 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
index 83d1192ecba6a..890f3e0ab2ffa 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith-usat.ll
@@ -84,22 +84,22 @@ define i32 @add(i32 %arg) {
;
; MVE-LABEL: 'add'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:122 CodeSize:79 Lat:122 SizeLat:122 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:158 Lat:244 SizeLat:244 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:488 CodeSize:316 Lat:488 SizeLat:488 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -215,22 +215,22 @@ define i32 @sub(i32 %arg) {
;
; MVE-LABEL: 'sub'
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:8 SizeLat:8 for: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:122 CodeSize:79 Lat:122 SizeLat:122 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:158 Lat:244 SizeLat:244 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:488 CodeSize:316 Lat:488 SizeLat:488 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:7 Lat:5 SizeLat:5 for: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/cmps.ll b/llvm/test/Analysis/CostModel/ARM/cmps.ll
index 5a648f82611bb..efcbb86df8e3f 100644
--- a/llvm/test/Analysis/CostModel/ARM/cmps.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cmps.ll
@@ -20,7 +20,7 @@ define i32 @cmps() {
; CHECK-MVE-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a10 = fcmp olt <8 x half> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:24 CodeSize:12 Lat:24 SizeLat:24 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:28 CodeSize:14 Lat:28 SizeLat:28 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
@@ -30,17 +30,17 @@ define i32 @cmps() {
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 48 for: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 24 for: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 64 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 32 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %g = icmp sge <4 x i32> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 8 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 32 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; CHECK-V8M-BASE-LABEL: 'cmps'
@@ -48,17 +48,17 @@ define i32 @cmps() {
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 48 for: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 24 for: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 64 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 32 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %g = icmp sge <4 x i32> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 8 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 32 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; CHECK-V8R-LABEL: 'cmps'
@@ -72,9 +72,9 @@ define i32 @cmps() {
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
-; CHECK-V8R-NEXT: Cost Model: Found costs of 48 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8R-NEXT: Cost Model: Found costs of 64 for: %a10 = fcmp olt <8 x half> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8R-NEXT: Cost Model: Found costs of 10 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8R-NEXT: Cost Model: Found costs of 12 for: %a12 = fcmp oge <2 x double> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of 1 for: %q = icmp eq <4 x ptr> undef, undef
; CHECK-V8R-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
@@ -121,10 +121,10 @@ define void @minmax() {
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 12 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 16 for: %c6 = icmp slt <4 x ptr> undef, undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
; CHECK-V8M-MAIN-NEXT: Cost Model: Found costs of 1 for: ret void
;
@@ -136,10 +136,10 @@ define void @minmax() {
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:20 CodeSize:24 Lat:20 SizeLat:20 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 12 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 16 for: %c6 = icmp slt <4 x ptr> undef, undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
; CHECK-V8M-BASE-NEXT: Cost Model: Found costs of 1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll b/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
index a96b040eacbe5..ceeda671a8875 100644
--- a/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
+++ b/llvm/test/Analysis/CostModel/ARM/fptoi_sat.ll
@@ -137,15 +137,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f32u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f32(<2 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:76 CodeSize:5 Lat:9 SizeLat:9 for: %v2f32s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f32(<2 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:72 CodeSize:3 Lat:5 SizeLat:5 for: %v2f32u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f32(<2 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:96 CodeSize:43 Lat:61 SizeLat:61 for: %v2f64s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:100 CodeSize:45 Lat:65 SizeLat:65 for: %v2f64s1 = call <2 x i1> @llvm.fptosi.sat.v2i1.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u1 = call <2 x i1> @llvm.fptoui.sat.v2i1.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:82 CodeSize:32 Lat:47 SizeLat:47 for: %v2f64s8 = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u8 = call <2 x i8> @llvm.fptoui.sat.v2i8.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:82 CodeSize:32 Lat:47 SizeLat:47 for: %v2f64s16 = call <2 x i16> @llvm.fptosi.sat.v2i16.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u16 = call <2 x i16> @llvm.fptoui.sat.v2i16.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:78 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:82 CodeSize:32 Lat:47 SizeLat:47 for: %v2f64s32 = call <2 x i32> @llvm.fptosi.sat.v2i32.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:52 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u32 = call <2 x i32> @llvm.fptoui.sat.v2i32.v2f64(<2 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:110 CodeSize:30 Lat:43 SizeLat:43 for: %v2f64s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:114 CodeSize:32 Lat:47 SizeLat:47 for: %v2f64s64 = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:84 CodeSize:17 Lat:17 SizeLat:17 for: %v2f64u64 = call <2 x i64> @llvm.fptoui.sat.v2i64.v2f64(<2 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:6 CodeSize:7 Lat:6 SizeLat:6 for: %v4f32s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:6 CodeSize:7 Lat:6 SizeLat:6 for: %v4f32u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f32(<4 x float> undef)
@@ -157,15 +157,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4f32u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:284 CodeSize:6 Lat:11 SizeLat:11 for: %v4f32s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f32(<4 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:278 CodeSize:3 Lat:5 SizeLat:5 for: %v4f32u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f32(<4 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:266 CodeSize:85 Lat:121 SizeLat:121 for: %v4f64s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:274 CodeSize:89 Lat:129 SizeLat:129 for: %v4f64s1 = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u1 = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:236 CodeSize:62 Lat:91 SizeLat:91 for: %v4f64s8 = call <4 x i8> @llvm.fptosi.sat.v4i8.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u8 = call <4 x i8> @llvm.fptoui.sat.v4i8.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:236 CodeSize:62 Lat:91 SizeLat:91 for: %v4f64s16 = call <4 x i16> @llvm.fptosi.sat.v4i16.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u16 = call <4 x i16> @llvm.fptoui.sat.v4i16.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:228 CodeSize:58 Lat:83 SizeLat:83 for: %v4f64s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:236 CodeSize:62 Lat:91 SizeLat:91 for: %v4f64s32 = call <4 x i32> @llvm.fptosi.sat.v4i32.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:178 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f64(<4 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:356 CodeSize:59 Lat:85 SizeLat:85 for: %v4f64s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:364 CodeSize:63 Lat:93 SizeLat:93 for: %v4f64s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:304 CodeSize:33 Lat:33 SizeLat:33 for: %v4f64u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f64(<4 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %v8f32s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:12 SizeLat:12 for: %v8f32u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f32(<8 x float> undef)
@@ -177,15 +177,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %v8f32u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1148 CodeSize:43 Lat:53 SizeLat:53 for: %v8f32s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f32(<8 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1104 CodeSize:5 Lat:9 SizeLat:9 for: %v8f32u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f32(<8 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:826 CodeSize:169 Lat:241 SizeLat:241 for: %v8f64s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:842 CodeSize:177 Lat:257 SizeLat:257 for: %v8f64s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:114 Lat:163 SizeLat:163 for: %v8f64s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:764 CodeSize:122 Lat:179 SizeLat:179 for: %v8f64s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:114 Lat:163 SizeLat:163 for: %v8f64s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:764 CodeSize:122 Lat:179 SizeLat:179 for: %v8f64s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:650 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:748 CodeSize:115 Lat:165 SizeLat:165 for: %v8f64s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:764 CodeSize:123 Lat:181 SizeLat:181 for: %v8f64s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:648 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f64(<8 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1256 CodeSize:117 Lat:169 SizeLat:169 for: %v8f64s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1272 CodeSize:125 Lat:185 SizeLat:185 for: %v8f64s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:1152 CodeSize:65 Lat:65 SizeLat:65 for: %v8f64u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f64(<8 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %v16f32s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:24 SizeLat:24 for: %v16f32u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f32(<16 x float> undef)
@@ -197,15 +197,15 @@ define void @casts() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v16f32u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4488 CodeSize:85 Lat:105 SizeLat:105 for: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4400 CodeSize:9 Lat:17 SizeLat:17 for: %v16f32u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f32(<16 x float> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2826 CodeSize:337 Lat:481 SizeLat:481 for: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2858 CodeSize:353 Lat:513 SizeLat:513 for: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2474 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2668 CodeSize:226 Lat:323 SizeLat:323 for: %v16f64s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2700 CodeSize:242 Lat:355 SizeLat:355 for: %v16f64s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2474 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2668 CodeSize:227 Lat:325 SizeLat:325 for: %v16f64s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2700 CodeSize:243 Lat:357 SizeLat:357 for: %v16f64s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2472 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2664 CodeSize:229 Lat:329 SizeLat:329 for: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2696 CodeSize:245 Lat:361 SizeLat:361 for: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2464 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f64(<16 x double> undef)
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4688 CodeSize:233 Lat:337 SizeLat:337 for: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4720 CodeSize:249 Lat:369 SizeLat:369 for: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:4480 CodeSize:129 Lat:129 SizeLat:129 for: %v16f64u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f64(<16 x double> undef)
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
index 5b8840c246d99..ed5242b758893 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-abs.ll
@@ -32,22 +32,22 @@ declare <64 x i8> @llvm.abs.v64i8(<64 x i8>, i1)
define i32 @abs(i32 %arg) {
; MVE-LABEL: 'abs'
; MVE-NEXT: Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.abs.i64(i64 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:63 Lat:90 SizeLat:90 for: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:180 CodeSize:126 Lat:180 SizeLat:180 for: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:360 CodeSize:252 Lat:360 SizeLat:360 for: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:122 CodeSize:79 Lat:122 SizeLat:122 for: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:158 Lat:244 SizeLat:244 for: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:488 CodeSize:316 Lat:488 SizeLat:488 for: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.abs.i32(i32 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.abs.i16(i16 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.abs.i8(i8 undef, i1 false)
-; MVE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:37 Lat:56 SizeLat:56 for: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
+; MVE-NEXT: Cost Model: Found costs of RThru:72 CodeSize:45 Lat:72 SizeLat:72 for: %V2I8 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
index 6e80420152588..df2f492aae9a4 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-cmp.ll
@@ -6,24 +6,24 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define void @icmp() {
; CHECK-LABEL: 'icmp'
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i8 = icmp slt <2 x i8> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:52 CodeSize:26 Lat:52 SizeLat:52 for: %v2i8 = icmp slt <2 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i8 = icmp slt <4 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v8i8 = icmp slt <8 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v16i8 = icmp slt <16 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:132 CodeSize:130 Lat:132 SizeLat:132 for: %v32i8 = icmp slt <32 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i16 = icmp slt <2 x i16> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:52 CodeSize:26 Lat:52 SizeLat:52 for: %v2i16 = icmp slt <2 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i16 = icmp slt <4 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v8i16 = icmp slt <8 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:68 CodeSize:66 Lat:68 SizeLat:68 for: %v16i16 = icmp slt <16 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:18 Lat:36 SizeLat:36 for: %v2i32 = icmp slt <2 x i32> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:52 CodeSize:26 Lat:52 SizeLat:52 for: %v2i32 = icmp slt <2 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i32 = icmp slt <4 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:36 CodeSize:34 Lat:36 SizeLat:36 for: %v8i32 = icmp slt <8 x i32> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:72 CodeSize:68 Lat:72 SizeLat:72 for: %v16i32 = icmp slt <16 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:52 CodeSize:26 Lat:52 SizeLat:52 for: %v2i64 = icmp slt <2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:104 CodeSize:52 Lat:104 SizeLat:104 for: %v4i64 = icmp slt <4 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:208 CodeSize:104 Lat:208 SizeLat:208 for: %v8i64 = icmp slt <8 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:84 CodeSize:42 Lat:84 SizeLat:84 for: %v2i128 = icmp slt <2 x i128> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:168 CodeSize:84 Lat:168 SizeLat:168 for: %v4i128 = icmp slt <4 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:84 CodeSize:42 Lat:84 SizeLat:84 for: %v2i64 = icmp slt <2 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:168 CodeSize:84 Lat:168 SizeLat:168 for: %v4i64 = icmp slt <4 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:336 CodeSize:168 Lat:336 SizeLat:336 for: %v8i64 = icmp slt <8 x i64> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:148 CodeSize:74 Lat:148 SizeLat:148 for: %v2i128 = icmp slt <2 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:296 CodeSize:148 Lat:296 SizeLat:296 for: %v4i128 = icmp slt <4 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2i8 = icmp slt <2 x i8> undef, undef
@@ -76,9 +76,9 @@ define void @fcmp() {
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4f32 = fcmp olt <4 x float> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:36 CodeSize:34 Lat:36 SizeLat:36 for: %v8f32 = fcmp olt <8 x float> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:72 CodeSize:68 Lat:72 SizeLat:72 for: %v16f32 = fcmp olt <16 x float> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:24 CodeSize:12 Lat:24 SizeLat:24 for: %v2f64 = fcmp olt <2 x double> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:48 CodeSize:24 Lat:48 SizeLat:48 for: %v4f64 = fcmp olt <4 x double> undef, undef
-; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:96 CodeSize:48 Lat:96 SizeLat:96 for: %v8f64 = fcmp olt <8 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:28 CodeSize:14 Lat:28 SizeLat:28 for: %v2f64 = fcmp olt <2 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:56 CodeSize:28 Lat:56 SizeLat:56 for: %v4f64 = fcmp olt <4 x double> undef, undef
+; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:112 CodeSize:56 Lat:112 SizeLat:112 for: %v8f64 = fcmp olt <8 x double> undef, undef
; CHECK-MVEFP-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f16 = fcmp olt <2 x half> undef, undef
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
index af6c694bb42f9..d32f726541b4c 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-minmax.ll
@@ -33,22 +33,22 @@ declare <64 x i8> @llvm.smin.v64i8(<64 x i8>, <64 x i8>)
define i32 @smin(i32 %arg) {
; MVE-LABEL: 'smin'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:86 CodeSize:43 Lat:86 SizeLat:86 for: %V2I64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:172 CodeSize:86 Lat:172 SizeLat:172 for: %V4I64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:344 CodeSize:172 Lat:344 SizeLat:344 for: %V8I64 = call <8 x i64> @llvm.smin.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.smin.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.smin.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I8 = call <2 x i8> @llvm.smin.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -115,22 +115,22 @@ declare <64 x i8> @llvm.smax.v64i8(<64 x i8>, <64 x i8>)
define i32 @smax(i32 %arg) {
; MVE-LABEL: 'smax'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:86 CodeSize:43 Lat:86 SizeLat:86 for: %V2I64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:172 CodeSize:86 Lat:172 SizeLat:172 for: %V4I64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:344 CodeSize:172 Lat:344 SizeLat:344 for: %V8I64 = call <8 x i64> @llvm.smax.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.smax.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.smax.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I8 = call <2 x i8> @llvm.smax.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -198,22 +198,22 @@ declare <64 x i8> @llvm.umin.v64i8(<64 x i8>, <64 x i8>)
define i32 @umin(i32 %arg) {
; MVE-LABEL: 'umin'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:86 CodeSize:43 Lat:86 SizeLat:86 for: %V2I64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:172 CodeSize:86 Lat:172 SizeLat:172 for: %V4I64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:344 CodeSize:172 Lat:344 SizeLat:344 for: %V8I64 = call <8 x i64> @llvm.umin.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.umin.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.umin.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I8 = call <2 x i8> @llvm.umin.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
@@ -280,22 +280,22 @@ declare <64 x i8> @llvm.umax.v64i8(<64 x i8>, <64 x i8>)
define i32 @umax(i32 %arg) {
; MVE-LABEL: 'umax'
; MVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %I64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:108 CodeSize:54 Lat:108 SizeLat:108 for: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:216 CodeSize:108 Lat:216 SizeLat:216 for: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:86 CodeSize:43 Lat:86 SizeLat:86 for: %V2I64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:172 CodeSize:86 Lat:172 SizeLat:172 for: %V4I64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:344 CodeSize:172 Lat:344 SizeLat:344 for: %V8I64 = call <8 x i64> @llvm.umax.v8i64(<8 x i64> undef, <8 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8I32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16I32 = call <16 x i32> @llvm.umax.v16i32(<16 x i32> undef, <16 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V32I16 = call <32 x i16> @llvm.umax.v32i16(<32 x i16> undef, <32 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %I8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:38 CodeSize:19 Lat:38 SizeLat:38 for: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:54 CodeSize:27 Lat:54 SizeLat:54 for: %V2I8 = call <2 x i8> @llvm.umax.v2i8(<2 x i8> undef, <2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4I8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8I8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
index 98d07ac71b42e..e660dc183b421 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smax.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:17 Lat:16 SizeLat:16 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:44 CodeSize:47 Lat:44 SizeLat:44 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:100 CodeSize:107 Lat:100 SizeLat:100 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:212 CodeSize:227 Lat:212 SizeLat:212 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:83 Lat:158 SizeLat:158 for: %V2 = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:308 CodeSize:158 Lat:308 SizeLat:308 for: %V4 = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:608 CodeSize:308 Lat:608 SizeLat:608 for: %V8 = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1208 CodeSize:608 Lat:1208 SizeLat:1208 for: %V16 = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:1017 CodeSize:1144 Lat:1017 SizeLat:1017 for: %V128 = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
index db8ca68723414..db90121fac4d7 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-smin.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:17 Lat:16 SizeLat:16 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:44 CodeSize:47 Lat:44 SizeLat:44 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:100 CodeSize:107 Lat:100 SizeLat:100 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:212 CodeSize:227 Lat:212 SizeLat:212 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:83 Lat:158 SizeLat:158 for: %V2 = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:308 CodeSize:158 Lat:308 SizeLat:308 for: %V4 = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:608 CodeSize:308 Lat:608 SizeLat:608 for: %V8 = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1208 CodeSize:608 Lat:1208 SizeLat:1208 for: %V16 = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:1017 CodeSize:1144 Lat:1017 SizeLat:1017 for: %V128 = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
index affea95302336..9b03a55cac35c 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umax.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:17 Lat:16 SizeLat:16 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:44 CodeSize:47 Lat:44 SizeLat:44 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:100 CodeSize:107 Lat:100 SizeLat:100 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:212 CodeSize:227 Lat:212 SizeLat:212 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:83 Lat:158 SizeLat:158 for: %V2 = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:308 CodeSize:158 Lat:308 SizeLat:308 for: %V4 = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:608 CodeSize:308 Lat:608 SizeLat:608 for: %V8 = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1208 CodeSize:608 Lat:1208 SizeLat:1208 for: %V16 = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:1017 CodeSize:1144 Lat:1017 SizeLat:1017 for: %V128 = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
index 6f3f025aef7a7..f6a5cc0a2713f 100644
--- a/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
+++ b/llvm/test/Analysis/CostModel/ARM/reduce-umin.ll
@@ -8,10 +8,10 @@ target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
define i32 @reduce_i64(i32 %arg) {
; V8M-LABEL: 'reduce_i64'
; V8M-NEXT: Cost Model: Found costs of 2 for: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:14 CodeSize:15 Lat:14 SizeLat:14 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:38 CodeSize:41 Lat:38 SizeLat:38 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:86 CodeSize:93 Lat:86 SizeLat:86 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:182 CodeSize:197 Lat:182 SizeLat:182 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:16 CodeSize:17 Lat:16 SizeLat:16 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:44 CodeSize:47 Lat:44 SizeLat:44 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:100 CodeSize:107 Lat:100 SizeLat:100 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:212 CodeSize:227 Lat:212 SizeLat:212 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i64'
@@ -24,10 +24,10 @@ define i32 @reduce_i64(i32 %arg) {
;
; MVE-LABEL: 'reduce_i64'
; MVE-NEXT: Cost Model: Found costs of 8 for: %V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:126 CodeSize:67 Lat:126 SizeLat:126 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:244 CodeSize:126 Lat:244 SizeLat:244 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:244 Lat:480 SizeLat:480 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
-; MVE-NEXT: Cost Model: Found costs of RThru:952 CodeSize:480 Lat:952 SizeLat:952 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:158 CodeSize:83 Lat:158 SizeLat:158 for: %V2 = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:308 CodeSize:158 Lat:308 SizeLat:308 for: %V4 = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:608 CodeSize:308 Lat:608 SizeLat:608 for: %V8 = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:1208 CodeSize:608 Lat:1208 SizeLat:1208 for: %V16 = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%V1 = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> undef)
@@ -40,11 +40,11 @@ define i32 @reduce_i64(i32 %arg) {
define i32 @reduce_i32(i32 %arg) {
; V8M-LABEL: 'reduce_i32'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i32'
@@ -56,7 +56,7 @@ define i32 @reduce_i32(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i32'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V8 = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %V16 = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> undef)
@@ -73,12 +73,12 @@ define i32 @reduce_i32(i32 %arg) {
define i32 @reduce_i16(i32 %arg) {
; V8M-LABEL: 'reduce_i16'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i16'
@@ -91,7 +91,7 @@ define i32 @reduce_i16(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i16'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %V16 = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> undef)
@@ -110,13 +110,13 @@ define i32 @reduce_i16(i32 %arg) {
define i32 @reduce_i8(i32 %arg) {
; V8M-LABEL: 'reduce_i8'
-; V8M-NEXT: Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:8 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:22 CodeSize:25 Lat:22 SizeLat:22 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:50 CodeSize:57 Lat:50 SizeLat:50 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:106 CodeSize:121 Lat:106 SizeLat:106 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:218 CodeSize:249 Lat:218 SizeLat:218 for: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:442 CodeSize:505 Lat:442 SizeLat:442 for: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
-; V8M-NEXT: Cost Model: Found costs of RThru:890 CodeSize:1017 Lat:890 SizeLat:890 for: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:9 CodeSize:10 Lat:9 SizeLat:9 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:25 CodeSize:28 Lat:25 SizeLat:25 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:57 CodeSize:64 Lat:57 SizeLat:57 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:121 CodeSize:136 Lat:121 SizeLat:121 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:249 CodeSize:280 Lat:249 SizeLat:249 for: %V32 = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:505 CodeSize:568 Lat:505 SizeLat:505 for: %V64 = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> undef)
+; V8M-NEXT: Cost Model: Found costs of RThru:1017 CodeSize:1144 Lat:1017 SizeLat:1017 for: %V128 = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> undef)
; V8M-NEXT: Cost Model: Found costs of 1 for: ret i32 undef
;
; NEON-LABEL: 'reduce_i8'
@@ -130,7 +130,7 @@ define i32 @reduce_i8(i32 %arg) {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; MVE-LABEL: 'reduce_i8'
-; MVE-NEXT: Cost Model: Found costs of RThru:74 CodeSize:39 Lat:74 SizeLat:74 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
+; MVE-NEXT: Cost Model: Found costs of RThru:90 CodeSize:47 Lat:90 SizeLat:90 for: %V2 = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V4 = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %V8 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> undef)
; MVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll b/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
index 80abbdd1cd90b..409d665226854 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/cmp-expanded.ll
@@ -6,7 +6,7 @@ define void @exts() {
; VSX is disabled, so this cost needs to include scalarization (because
; <4 x double> is legalized to scalars).
- ; CHECK: cost of 56 {{.*}} fcmp
+ ; CHECK: cost of 68 {{.*}} fcmp
%v1 = fcmp ugt <4 x double> undef, undef
ret void
diff --git a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
index f19cdc759b495..8b76ab75173e4 100644
--- a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
+++ b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
@@ -877,25 +877,25 @@ define void @fp16() {
; SSE2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 114 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 104 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 111 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 79 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 103 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 110 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 242 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 216 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 230 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 221 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 235 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 215 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 229 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SSE2-NEXT: Cost Model: Found an estimated cost of 219 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 233 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -930,25 +930,25 @@ define void @fp16() {
; SSE42-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 114 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 77 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 103 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 110 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 102 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 213 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 227 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 201 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 215 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 206 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 220 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 200 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 214 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SSE42-NEXT: Cost Model: Found an estimated cost of 204 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SSE42-NEXT: Cost Model: Found an estimated cost of 218 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SSE42-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -983,25 +983,25 @@ define void @fp16() {
; AVX2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 84 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 91 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 59 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 113 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 120 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 111 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 118 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 86 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 116 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 84 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 219 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 234 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 170 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 243 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 179 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 228 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 243 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 179 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 226 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 241 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 175 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; AVX2-NEXT: Cost Model: Found an estimated cost of 208 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 223 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -1142,25 +1142,25 @@ define void @fp16() {
; SLM-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %v4f16u32 = call <4 x i32> @llvm.fptoui.sat.v4i32.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %v4f16s64 = call <4 x i64> @llvm.fptosi.sat.v4i64.v4f16(<4 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v4f16u64 = call <4 x i64> @llvm.fptoui.sat.v4i64.v4f16(<4 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 109 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 116 for instruction: %v8f16s1 = call <8 x i1> @llvm.fptosi.sat.v8i1.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 82 for instruction: %v8f16u1 = call <8 x i1> @llvm.fptoui.sat.v8i1.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 104 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 111 for instruction: %v8f16s8 = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 77 for instruction: %v8f16u8 = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 105 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %v8f16s16 = call <8 x i16> @llvm.fptosi.sat.v8i16.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %v8f16u16 = call <8 x i16> @llvm.fptoui.sat.v8i16.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 107 for instruction: %v8f16s32 = call <8 x i32> @llvm.fptosi.sat.v8i32.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 73 for instruction: %v8f16u32 = call <8 x i32> @llvm.fptoui.sat.v8i32.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 110 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 117 for instruction: %v8f16s64 = call <8 x i64> @llvm.fptosi.sat.v8i64.v8f16(<8 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 71 for instruction: %v8f16u64 = call <8 x i64> @llvm.fptoui.sat.v8i64.v8f16(<8 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 215 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 229 for instruction: %v16f16s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 165 for instruction: %v16f16u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 203 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 217 for instruction: %v16f16s8 = call <16 x i8> @llvm.fptosi.sat.v16i8.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 153 for instruction: %v16f16u8 = call <16 x i8> @llvm.fptoui.sat.v16i8.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 210 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 224 for instruction: %v16f16s16 = call <16 x i16> @llvm.fptosi.sat.v16i16.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 156 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 200 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 214 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 146 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; SLM-NEXT: Cost Model: Found an estimated cost of 220 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; SLM-NEXT: Cost Model: Found an estimated cost of 234 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 142 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
index dd7352ef2b26b..e2300a0f48148 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
@@ -27,7 +27,7 @@ define void @expensive_icmp(ptr noalias nocapture %d, ptr nocapture readonly %s,
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK: Cost of 18 for VF 2: WIDEN ir<%1> = load vp<[[VP5]]>
; CHECK: Cost of 4 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%1> to i32
-; CHECK: Cost of 36 for VF 2: WIDEN ir<%cmp2> = icmp sgt ir<%conv>, ir<%conv1>
+; CHECK: Cost of 52 for VF 2: WIDEN ir<%cmp2> = icmp sgt ir<%conv>, ir<%conv1>
; CHECK: Cost of 26 for VF 2: WIDEN ir<%conv6> = add ir<%1>, ir<%0>
; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx7> = getelementptr ir<%d>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%arrayidx7>, ir<1>
@@ -45,7 +45,7 @@ define void @expensive_icmp(ptr noalias nocapture %d, ptr nocapture readonly %s,
; CHECK: Cost of 0 for VF 2: IR %cmp2 = icmp sgt i32 %conv, %conv1
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost for VF 2: 102 (Estimated cost per lane: 51)
+; CHECK: Cost for VF 2: 118 (Estimated cost per lane: 59)
; CHECK: Cost of 1 for VF 4: induction instruction %inc = add nuw nsw i32 %i.016, 1
; CHECK: Cost of 0 for VF 4: induction instruction %i.016 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
@@ -179,7 +179,7 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 26 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv3>, ir<%conv1>
; CHECK: Cost of 18 for VF 2: WIDEN ir<%shr> = ashr ir<%mul>, ir<7>
; CHECK: Cost of 0 for VF 2: WIDEN ir<%2> = icmp slt ir<%shr>, ir<127>
-; CHECK: Cost of 38 for VF 2: WIDEN ir<%spec.select.i> = select ir<%2>, ir<%shr>, ir<127>
+; CHECK: Cost of 54 for VF 2: WIDEN ir<%spec.select.i> = select ir<%2>, ir<%shr>, ir<127>
; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%conv4> = trunc ir<%spec.select.i> to i8
; CHECK: Cost of 0 for VF 2: vp<[[VP13:%[0-9]+]]> = vector-pointer vp<%next.gep>.1, ir<1>
; CHECK: Cost of 18 for VF 2: WIDEN store vp<[[VP13]]>, ir<%conv4>
@@ -215,7 +215,7 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 0 for VF 2: vp<[[VP6]]> = DERIVED-IV ir<%pSrcB> + vp<[[VP2]]> * ir<1>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost for VF 2: 146 (Estimated cost per lane: 73)
+; CHECK: Cost for VF 2: 162 (Estimated cost per lane: 81)
; CHECK: Cost of 1 for VF 4: induction instruction %dec = add i32 %blkCnt.012, -1
; CHECK: Cost of 0 for VF 4: induction instruction %blkCnt.012 = phi i32 [ %dec, %while.body ], [ %blockSize, %while.body.preheader ]
; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
>From 31a77a8933b8ba1e9cc93f6fefc1ede952689cdb Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 22 Jul 2026 23:09:27 +0800
Subject: [PATCH 7/7] Update SLPVectorizer tests
---
.../test/Transforms/SLPVectorizer/AMDGPU/min_max.ll | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index 4b3e8f1487186..f9e1c764a52ed 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -357,20 +357,17 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
; GFX8-NEXT: bb:
; GFX8-NEXT: [[ARG0_0:%.*]] = extractelement <4 x i16> [[ARG0:%.*]], i64 0
; GFX8-NEXT: [[ARG0_1:%.*]] = extractelement <4 x i16> [[ARG0]], i64 1
-; GFX8-NEXT: [[ARG0_2:%.*]] = extractelement <4 x i16> [[ARG0]], i64 2
-; GFX8-NEXT: [[ARG0_3:%.*]] = extractelement <4 x i16> [[ARG0]], i64 3
; GFX8-NEXT: [[ARG1_0:%.*]] = extractelement <4 x i16> [[ARG1:%.*]], i64 0
; GFX8-NEXT: [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
-; GFX8-NEXT: [[ARG1_2:%.*]] = extractelement <4 x i16> [[ARG1]], i64 2
-; GFX8-NEXT: [[ARG1_3:%.*]] = extractelement <4 x i16> [[ARG1]], i64 3
; GFX8-NEXT: [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
; GFX8-NEXT: [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT: [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT: [[ADD_3:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
+; GFX8-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP3]])
; GFX8-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
; GFX8-NEXT: [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
-; GFX8-NEXT: [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
-; GFX8-NEXT: [[INS_31:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
+; GFX8-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT: [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; GFX8-NEXT: ret <4 x i16> [[INS_31]]
;
; GFX9-LABEL: @uadd_sat_v4i16(
More information about the llvm-commits
mailing list