[llvm] [AArch64] Give a higher cost for more expensive SVE FCMP instructions (PR #153816)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 15 07:56:59 PDT 2025
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/153816
>From 118d1bcfb2562f9144faf22b9daa67f678645a78 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 24 Jun 2025 19:15:54 +0100
Subject: [PATCH 1/2] [AArch64] Add HasHalfSVEFCmpThroughput
>From the SWOG's for various CPUs, SVE fcmp can have a lower throughput that the
Neon instructions. This adds a cost-model tuning feature to the relevant CPUs
and increase the costs in those cases. This should lead to fixed-width being
slightly more prefered on those CPUs.
In the long run this information would ideally come straight from the
scheduling model, but that is currently unproven and my experiments with it
have ran into issues that are not yet addressed.
---
llvm/lib/Target/AArch64/AArch64Features.td | 5 +
llvm/lib/Target/AArch64/AArch64Processors.td | 14 +
.../AArch64/AArch64TargetTransformInfo.cpp | 4 +
.../Analysis/CostModel/AArch64/sve-cmpsel.ll | 10 +-
.../Analysis/CostModel/AArch64/sve-fcmp.ll | 394 +++++++++---------
.../SLPVectorizer/AArch64/slp-fma-loss.ll | 14 +-
6 files changed, 232 insertions(+), 209 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index c1c1f0a1024d0..9f995816b0e60 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -847,6 +847,11 @@ def FeatureUseWzrToVecMove : SubtargetFeature<"use-wzr-to-vec-move",
"UseWzrToVecMove", "true",
"Move from WZR to insert 0 into vector registers">;
+// SVE fcmxx operations sometimes have half the throughput of Neon cmxx.
+def FeatureHalfSVEFCmpThroughput : SubtargetFeature<"half-sve-fcmp-throughput",
+ "HasHalfSVEFCmpThroughput", "false",
+ "SVE FCmp instructions have half the throughput of Neon instructions">;
+
//===----------------------------------------------------------------------===//
// Architectures.
//
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 1bc1d98a6f65b..94b6aba7cf0cf 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -21,6 +21,7 @@ def TuneA320 : SubtargetFeature<"a320", "ARMProcFamily", "CortexA320",
"Cortex-A320 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
+ FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost]>;
@@ -45,6 +46,7 @@ def TuneA510 : SubtargetFeature<"a510", "ARMProcFamily", "CortexA510",
"Cortex-A510 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
+ FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost
@@ -54,6 +56,7 @@ def TuneA520 : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520",
"Cortex-A520 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
+ FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost]>;
@@ -62,6 +65,7 @@ def TuneA520AE : SubtargetFeature<"a520ae", "ARMProcFamily", "CortexA520",
"Cortex-A520AE ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
+ FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove]>;
@@ -182,6 +186,7 @@ def TuneA715 : SubtargetFeature<"a715", "ARMProcFamily", "CortexA715",
FeatureALULSLFast,
FeatureFuseAdrpAdd,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeaturePredictableSelectIsExpensive]>;
def TuneA720 : SubtargetFeature<"a720", "ARMProcFamily", "CortexA720",
@@ -244,6 +249,7 @@ def TuneX2 : SubtargetFeature<"cortex-x2", "ARMProcFamily", "CortexX2",
FeatureALULSLFast,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeatureUseFixedOverScalableIfEqualCost,
FeaturePredictableSelectIsExpensive]>;
@@ -254,6 +260,7 @@ def TuneX3 : SubtargetFeature<"cortex-x3", "ARMProcFamily", "CortexX3",
FeatureFuseAES,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeatureUseFixedOverScalableIfEqualCost,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -266,6 +273,7 @@ def TuneX4 : SubtargetFeature<"cortex-x4", "ARMProcFamily", "CortexX4",
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
+ FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -277,6 +285,7 @@ def TuneX925 : SubtargetFeature<"cortex-x925", "ARMProcFamily",
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
+ FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -539,6 +548,7 @@ def TuneNeoverseN2 : SubtargetFeature<"neoversen2", "ARMProcFamily", "NeoverseN2
FeatureALULSLFast,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeaturePredictableSelectIsExpensive]>;
def TuneNeoverseN3 : SubtargetFeature<"neoversen3", "ARMProcFamily", "NeoverseN3",
@@ -568,6 +578,7 @@ def TuneNeoverseV1 : SubtargetFeature<"neoversev1", "ARMProcFamily", "NeoverseV1
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeaturePredictableSelectIsExpensive,
+ FeatureHalfSVEFCmpThroughput,
FeatureNoSVEFPLD1R]>;
def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2",
@@ -579,6 +590,7 @@ def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
+ FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive,
FeatureDisableLatencySchedHeuristic]>;
@@ -590,6 +602,7 @@ def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3
FeatureFuseAdrpAdd,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -600,6 +613,7 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover
FeatureFuseAdrpAdd,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
+ FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 3042251cf754d..2cbd447f7c2d3 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4462,6 +4462,10 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
(VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
Factor = 3; // fcmxx+fcmyy+or
+ if (isa<ScalableVectorType>(ValTy) &&
+ CostKind == TTI::TCK_RecipThroughput && ST->hasHalfSVEFCmpThroughput())
+ Factor *= 2;
+
return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first);
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll b/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
index 0789707a53b90..5029ef974681d 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-cmpsel.ll
@@ -58,10 +58,10 @@ define <vscale x 32 x i1> @cmp_nxv32i1() {
; Check fcmp for legal FP vectors
define void @cmp_legal_fp() #0 {
; CHECK-LABEL: 'cmp_legal_fp'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %1 = fcmp oge <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %2 = fcmp oge <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %3 = fcmp oge <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %4 = fcmp oge <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %1 = fcmp oge <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %2 = fcmp oge <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %3 = fcmp oge <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %4 = fcmp oge <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%1 = fcmp oge <vscale x 2 x double> undef, undef
@@ -74,7 +74,7 @@ define void @cmp_legal_fp() #0 {
; Check fcmp for an illegal FP vector
define <vscale x 16 x i1> @cmp_nxv16f16() {
; CHECK-LABEL: 'cmp_nxv16f16'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %res = fcmp oge <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %res = fcmp oge <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 16 x i1> %res
;
%res = fcmp oge <vscale x 16 x half> undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-fcmp.ll b/llvm/test/Analysis/CostModel/AArch64/sve-fcmp.ll
index a14176cd6d532..c48ca20949c21 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-fcmp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-fcmp.ll
@@ -3,15 +3,15 @@
define void @fcmp_oeq(i32 %arg) {
; CHECK-LABEL: 'fcmp_oeq'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp oeq <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp oeq <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp oeq <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp oeq <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp oeq <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp oeq <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp oeq <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp oeq <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp oeq <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp oeq <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp oeq <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp oeq <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp oeq <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp oeq <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp oeq <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp oeq <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp oeq <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp oeq <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp oeq <vscale x 2 x float> undef, undef
@@ -28,9 +28,9 @@ define void @fcmp_oeq(i32 %arg) {
define void @fcmp_oeq_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_oeq_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp oeq <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp oeq <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp oeq <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp oeq <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp oeq <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp oeq <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp oeq <vscale x 2 x bfloat> undef, undef
@@ -41,15 +41,15 @@ define void @fcmp_oeq_bfloat(i32 %arg) {
define void @fcmp_ogt(i32 %arg) {
; CHECK-LABEL: 'fcmp_ogt'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ogt <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ogt <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ogt <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ogt <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ogt <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ogt <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ogt <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ogt <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ogt <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ogt <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ogt <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ogt <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ogt <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ogt <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ogt <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ogt <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ogt <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ogt <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ogt <vscale x 2 x float> undef, undef
@@ -66,9 +66,9 @@ define void @fcmp_ogt(i32 %arg) {
define void @fcmp_ogt_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ogt_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ogt <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ogt <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ogt <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ogt <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ogt <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ogt <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ogt <vscale x 2 x bfloat> undef, undef
@@ -79,15 +79,15 @@ define void @fcmp_ogt_bfloat(i32 %arg) {
define void @fcmp_oge(i32 %arg) {
; CHECK-LABEL: 'fcmp_oge'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp oge <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp oge <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp oge <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp oge <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp oge <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp oge <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp oge <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp oge <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp oge <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp oge <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp oge <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp oge <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp oge <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp oge <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp oge <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp oge <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp oge <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp oge <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp oge <vscale x 2 x float> undef, undef
@@ -104,9 +104,9 @@ define void @fcmp_oge(i32 %arg) {
define void @fcmp_oge_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_oge_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp oge <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp oge <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp oge <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp oge <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp oge <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp oge <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp oge <vscale x 2 x bfloat> undef, undef
@@ -117,15 +117,15 @@ define void @fcmp_oge_bfloat(i32 %arg) {
define void @fcmp_olt(i32 %arg) {
; CHECK-LABEL: 'fcmp_olt'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp olt <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp olt <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp olt <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp olt <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp olt <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp olt <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp olt <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp olt <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp olt <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp olt <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp olt <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp olt <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp olt <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp olt <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp olt <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp olt <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp olt <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp olt <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp olt <vscale x 2 x float> undef, undef
@@ -142,9 +142,9 @@ define void @fcmp_olt(i32 %arg) {
define void @fcmp_olt_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_olt_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp olt <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp olt <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp olt <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp olt <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp olt <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp olt <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp olt <vscale x 2 x bfloat> undef, undef
@@ -155,15 +155,15 @@ define void @fcmp_olt_bfloat(i32 %arg) {
define void @fcmp_ole(i32 %arg) {
; CHECK-LABEL: 'fcmp_ole'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ole <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ole <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ole <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ole <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ole <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ole <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ole <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ole <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ole <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ole <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ole <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ole <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ole <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ole <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ole <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ole <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ole <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ole <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ole <vscale x 2 x float> undef, undef
@@ -180,9 +180,9 @@ define void @fcmp_ole(i32 %arg) {
define void @fcmp_ole_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ole_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ole <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ole <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ole <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ole <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ole <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ole <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ole <vscale x 2 x bfloat> undef, undef
@@ -193,15 +193,15 @@ define void @fcmp_ole_bfloat(i32 %arg) {
define void @fcmp_one(i32 %arg) {
; CHECK-LABEL: 'fcmp_one'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f32 = fcmp one <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v4f32 = fcmp one <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v8f32 = fcmp one <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f64 = fcmp one <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v4f64 = fcmp one <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f16 = fcmp one <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v4f16 = fcmp one <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v8f16 = fcmp one <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v16f16 = fcmp one <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f32 = fcmp one <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v4f32 = fcmp one <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v8f32 = fcmp one <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f64 = fcmp one <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v4f64 = fcmp one <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f16 = fcmp one <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v4f16 = fcmp one <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v8f16 = fcmp one <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v16f16 = fcmp one <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp one <vscale x 2 x float> undef, undef
@@ -218,9 +218,9 @@ define void @fcmp_one(i32 %arg) {
define void @fcmp_one_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_one_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:8 SizeLat:5 for: %v2bf16 = fcmp one <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:8 SizeLat:5 for: %v4bf16 = fcmp one <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:15 CodeSize:9 Lat:9 SizeLat:9 for: %v8bf16 = fcmp one <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:8 SizeLat:5 for: %v2bf16 = fcmp one <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:8 SizeLat:5 for: %v4bf16 = fcmp one <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:21 CodeSize:9 Lat:9 SizeLat:9 for: %v8bf16 = fcmp one <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp one <vscale x 2 x bfloat> undef, undef
@@ -231,15 +231,15 @@ define void @fcmp_one_bfloat(i32 %arg) {
define void @fcmp_ord(i32 %arg) {
; CHECK-LABEL: 'fcmp_ord'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ord <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ord <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ord <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ord <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ord <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ord <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ord <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ord <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ord <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ord <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ord <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ord <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ord <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ord <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ord <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ord <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ord <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ord <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ord <vscale x 2 x float> undef, undef
@@ -256,9 +256,9 @@ define void @fcmp_ord(i32 %arg) {
define void @fcmp_ord_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ord_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ord <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ord <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ord <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ord <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ord <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ord <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ord <vscale x 2 x bfloat> undef, undef
@@ -269,15 +269,15 @@ define void @fcmp_ord_bfloat(i32 %arg) {
define void @fcmp_ueq(i32 %arg) {
; CHECK-LABEL: 'fcmp_ueq'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f32 = fcmp ueq <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v4f32 = fcmp ueq <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v8f32 = fcmp ueq <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f64 = fcmp ueq <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v4f64 = fcmp ueq <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v2f16 = fcmp ueq <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v4f16 = fcmp ueq <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:6 SizeLat:3 for: %v8f16 = fcmp ueq <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 6 for: %v16f16 = fcmp ueq <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f32 = fcmp ueq <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v4f32 = fcmp ueq <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v8f32 = fcmp ueq <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f64 = fcmp ueq <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v4f64 = fcmp ueq <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v2f16 = fcmp ueq <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v4f16 = fcmp ueq <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:3 for: %v8f16 = fcmp ueq <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %v16f16 = fcmp ueq <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ueq <vscale x 2 x float> undef, undef
@@ -294,9 +294,9 @@ define void @fcmp_ueq(i32 %arg) {
define void @fcmp_ueq_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ueq_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:8 SizeLat:5 for: %v2bf16 = fcmp ueq <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:8 SizeLat:5 for: %v4bf16 = fcmp ueq <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:15 CodeSize:9 Lat:9 SizeLat:9 for: %v8bf16 = fcmp ueq <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:8 SizeLat:5 for: %v2bf16 = fcmp ueq <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:5 Lat:8 SizeLat:5 for: %v4bf16 = fcmp ueq <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:21 CodeSize:9 Lat:9 SizeLat:9 for: %v8bf16 = fcmp ueq <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ueq <vscale x 2 x bfloat> undef, undef
@@ -307,15 +307,15 @@ define void @fcmp_ueq_bfloat(i32 %arg) {
define void @fcmp_ugt(i32 %arg) {
; CHECK-LABEL: 'fcmp_ugt'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ugt <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ugt <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ugt <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ugt <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ugt <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ugt <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ugt <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ugt <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ugt <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ugt <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ugt <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ugt <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ugt <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ugt <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ugt <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ugt <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ugt <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ugt <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ugt <vscale x 2 x float> undef, undef
@@ -332,9 +332,9 @@ define void @fcmp_ugt(i32 %arg) {
define void @fcmp_ugt_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ugt_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ugt <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ugt <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ugt <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ugt <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ugt <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ugt <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ugt <vscale x 2 x bfloat> undef, undef
@@ -345,15 +345,15 @@ define void @fcmp_ugt_bfloat(i32 %arg) {
define void @fcmp_uge(i32 %arg) {
; CHECK-LABEL: 'fcmp_uge'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp uge <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp uge <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp uge <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp uge <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp uge <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp uge <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp uge <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp uge <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp uge <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp uge <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp uge <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp uge <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp uge <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp uge <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp uge <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp uge <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp uge <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp uge <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp uge <vscale x 2 x float> undef, undef
@@ -370,9 +370,9 @@ define void @fcmp_uge(i32 %arg) {
define void @fcmp_uge_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_uge_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp uge <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp uge <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp uge <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp uge <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp uge <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp uge <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp uge <vscale x 2 x bfloat> undef, undef
@@ -383,15 +383,15 @@ define void @fcmp_uge_bfloat(i32 %arg) {
define void @fcmp_ult(i32 %arg) {
; CHECK-LABEL: 'fcmp_ult'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ult <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ult <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ult <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ult <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ult <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ult <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ult <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ult <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ult <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ult <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ult <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ult <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ult <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ult <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ult <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ult <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ult <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ult <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ult <vscale x 2 x float> undef, undef
@@ -408,9 +408,9 @@ define void @fcmp_ult(i32 %arg) {
define void @fcmp_ult_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ult_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ult <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ult <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ult <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ult <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ult <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ult <vscale x 8 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ult <vscale x 2 x bfloat> undef, undef
@@ -421,15 +421,15 @@ define void @fcmp_ult_bfloat(i32 %arg) {
define void @fcmp_ule(i32 %arg) {
; CHECK-LABEL: 'fcmp_ule'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ule <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ule <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp ule <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ule <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp ule <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ule <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ule <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ule <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp ule <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp ule <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp ule <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp ule <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp ule <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp ule <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp ule <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp ule <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp ule <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp ule <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp ule <vscale x 2 x float> undef, undef
@@ -446,10 +446,10 @@ define void @fcmp_ule(i32 %arg) {
define void @fcmp_ule_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_ule_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ule <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ule <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ule <vscale x 8 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp ule <vscale x 16 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp ule <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp ule <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp ule <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:26 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp ule <vscale x 16 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp ule <vscale x 2 x bfloat> undef, undef
@@ -461,15 +461,15 @@ define void @fcmp_ule_bfloat(i32 %arg) {
define void @fcmp_une(i32 %arg) {
; CHECK-LABEL: 'fcmp_une'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp une <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp une <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp une <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp une <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp une <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp une <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp une <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp une <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp une <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp une <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp une <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp une <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp une <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp une <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp une <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp une <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp une <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp une <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp une <vscale x 2 x float> undef, undef
@@ -486,10 +486,10 @@ define void @fcmp_une(i32 %arg) {
define void @fcmp_une_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_une_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp une <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp une <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp une <vscale x 8 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp une <vscale x 16 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp une <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp une <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp une <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:26 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp une <vscale x 16 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp une <vscale x 2 x bfloat> undef, undef
@@ -501,15 +501,15 @@ define void @fcmp_une_bfloat(i32 %arg) {
define void @fcmp_uno(i32 %arg) {
; CHECK-LABEL: 'fcmp_uno'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp uno <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp uno <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp uno <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp uno <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp uno <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp uno <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp uno <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp uno <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp uno <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp uno <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp uno <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp uno <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp uno <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp uno <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp uno <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp uno <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp uno <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp uno <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp uno <vscale x 2 x float> undef, undef
@@ -526,10 +526,10 @@ define void @fcmp_uno(i32 %arg) {
define void @fcmp_uno_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_uno_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp uno <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp uno <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp uno <vscale x 8 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp uno <vscale x 16 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp uno <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp uno <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp uno <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:26 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp uno <vscale x 16 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp uno <vscale x 2 x bfloat> undef, undef
@@ -541,15 +541,15 @@ define void @fcmp_uno_bfloat(i32 %arg) {
define void @fcmp_true(i32 %arg) {
; CHECK-LABEL: 'fcmp_true'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp true <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp true <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp true <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp true <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp true <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp true <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp true <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp true <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp true <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp true <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp true <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp true <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp true <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp true <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp true <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp true <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp true <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp true <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp true <vscale x 2 x float> undef, undef
@@ -566,10 +566,10 @@ define void @fcmp_true(i32 %arg) {
define void @fcmp_true_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_true_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp true <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp true <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp true <vscale x 8 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp true <vscale x 16 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp true <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp true <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp true <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:26 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp true <vscale x 16 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp true <vscale x 2 x bfloat> undef, undef
@@ -581,15 +581,15 @@ define void @fcmp_true_bfloat(i32 %arg) {
define void @fcmp_false(i32 %arg) {
; CHECK-LABEL: 'fcmp_false'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp false <vscale x 2 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp false <vscale x 4 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v8f32 = fcmp false <vscale x 8 x float> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp false <vscale x 2 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v4f64 = fcmp false <vscale x 4 x double> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp false <vscale x 2 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp false <vscale x 4 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp false <vscale x 8 x half> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %v16f16 = fcmp false <vscale x 16 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f32 = fcmp false <vscale x 2 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f32 = fcmp false <vscale x 4 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v8f32 = fcmp false <vscale x 8 x float> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f64 = fcmp false <vscale x 2 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v4f64 = fcmp false <vscale x 4 x double> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v2f16 = fcmp false <vscale x 2 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v4f16 = fcmp false <vscale x 4 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:1 for: %v8f16 = fcmp false <vscale x 8 x half> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %v16f16 = fcmp false <vscale x 16 x half> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2f32 = fcmp false <vscale x 2 x float> undef, undef
@@ -606,10 +606,10 @@ define void @fcmp_false(i32 %arg) {
define void @fcmp_false_bfloat(i32 %arg) {
; CHECK-LABEL: 'fcmp_false_bfloat'
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp false <vscale x 2 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp false <vscale x 4 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp false <vscale x 8 x bfloat> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:22 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp false <vscale x 16 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v2bf16 = fcmp false <vscale x 2 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:3 for: %v4bf16 = fcmp false <vscale x 4 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:13 CodeSize:5 Lat:5 SizeLat:5 for: %v8bf16 = fcmp false <vscale x 8 x bfloat> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:26 CodeSize:7 Lat:5 SizeLat:7 for: %v16bf16 = fcmp false <vscale x 16 x bfloat> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%v2bf16 = fcmp false <vscale x 2 x bfloat> undef, undef
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
index 03f67ecb3e695..a2f497729d4f1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
@@ -216,8 +216,8 @@ define void @slp_profitable_missing_fmf_nnans_only(ptr %A, ptr %B) {
define float @slp_not_profitable_in_loop(float %x, ptr %A) {
; CHECK-LABEL: @slp_not_profitable_in_loop(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_A_2]], align 4
+; CHECK-NEXT: [[A1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[L_2:%.*]] = load float, ptr [[A1]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
; CHECK-NEXT: [[L_3:%.*]] = load float, ptr [[A]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> <float poison, float 3.000000e+00>, float [[X:%.*]], i32 0
@@ -226,13 +226,13 @@ define float @slp_not_profitable_in_loop(float %x, ptr %A) {
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[RED:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[MUL12:%.*]] = fmul fast float 3.000000e+00, [[L_1]]
+; CHECK-NEXT: [[ADD:%.*]] = fmul fast float 3.000000e+00, [[L_2]]
; CHECK-NEXT: [[MUL16:%.*]] = fmul fast float 3.000000e+00, [[L_3]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[MUL12]], [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; CHECK-NEXT: [[ADD13:%.*]] = fadd fast float [[ADD]], [[TMP4]]
-; CHECK-NEXT: [[RED_NEXT]] = fadd fast float [[ADD13]], [[MUL16]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[ADD14:%.*]] = fadd fast float [[ADD13]], [[TMP5]]
+; CHECK-NEXT: [[RED_NEXT]] = fadd fast float [[ADD14]], [[MUL16]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 10
; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
>From 8ba917cab154fdc1309ae9fe69fe0c663efc72bb Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Fri, 15 Aug 2025 15:56:47 +0100
Subject: [PATCH 2/2] Grab it from scheduling info.
---
llvm/lib/Target/AArch64/AArch64Features.td | 5 ---
llvm/lib/Target/AArch64/AArch64Processors.td | 14 --------
.../AArch64/AArch64TargetTransformInfo.cpp | 32 ++++++++++++++++++-
.../AArch64/AArch64TargetTransformInfo.h | 5 +++
.../LoopVectorize/AArch64/masked-call.ll | 25 ++++++---------
5 files changed, 45 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 9f995816b0e60..c1c1f0a1024d0 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -847,11 +847,6 @@ def FeatureUseWzrToVecMove : SubtargetFeature<"use-wzr-to-vec-move",
"UseWzrToVecMove", "true",
"Move from WZR to insert 0 into vector registers">;
-// SVE fcmxx operations sometimes have half the throughput of Neon cmxx.
-def FeatureHalfSVEFCmpThroughput : SubtargetFeature<"half-sve-fcmp-throughput",
- "HasHalfSVEFCmpThroughput", "false",
- "SVE FCmp instructions have half the throughput of Neon instructions">;
-
//===----------------------------------------------------------------------===//
// Architectures.
//
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 94b6aba7cf0cf..1bc1d98a6f65b 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -21,7 +21,6 @@ def TuneA320 : SubtargetFeature<"a320", "ARMProcFamily", "CortexA320",
"Cortex-A320 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
- FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost]>;
@@ -46,7 +45,6 @@ def TuneA510 : SubtargetFeature<"a510", "ARMProcFamily", "CortexA510",
"Cortex-A510 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
- FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost
@@ -56,7 +54,6 @@ def TuneA520 : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520",
"Cortex-A520 ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
- FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost]>;
@@ -65,7 +62,6 @@ def TuneA520AE : SubtargetFeature<"a520ae", "ARMProcFamily", "CortexA520",
"Cortex-A520AE ARM processors", [
FeatureFuseAES,
FeatureFuseAdrpAdd,
- FeatureHalfSVEFCmpThroughput,
FeaturePostRAScheduler,
FeatureUseWzrToVecMove]>;
@@ -186,7 +182,6 @@ def TuneA715 : SubtargetFeature<"a715", "ARMProcFamily", "CortexA715",
FeatureALULSLFast,
FeatureFuseAdrpAdd,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeaturePredictableSelectIsExpensive]>;
def TuneA720 : SubtargetFeature<"a720", "ARMProcFamily", "CortexA720",
@@ -249,7 +244,6 @@ def TuneX2 : SubtargetFeature<"cortex-x2", "ARMProcFamily", "CortexX2",
FeatureALULSLFast,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeatureUseFixedOverScalableIfEqualCost,
FeaturePredictableSelectIsExpensive]>;
@@ -260,7 +254,6 @@ def TuneX3 : SubtargetFeature<"cortex-x3", "ARMProcFamily", "CortexX3",
FeatureFuseAES,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeatureUseFixedOverScalableIfEqualCost,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -273,7 +266,6 @@ def TuneX4 : SubtargetFeature<"cortex-x4", "ARMProcFamily", "CortexX4",
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
- FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -285,7 +277,6 @@ def TuneX925 : SubtargetFeature<"cortex-x925", "ARMProcFamily",
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
- FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -548,7 +539,6 @@ def TuneNeoverseN2 : SubtargetFeature<"neoversen2", "ARMProcFamily", "NeoverseN2
FeatureALULSLFast,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeaturePredictableSelectIsExpensive]>;
def TuneNeoverseN3 : SubtargetFeature<"neoversen3", "ARMProcFamily", "NeoverseN3",
@@ -578,7 +568,6 @@ def TuneNeoverseV1 : SubtargetFeature<"neoversev1", "ARMProcFamily", "NeoverseV1
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeaturePredictableSelectIsExpensive,
- FeatureHalfSVEFCmpThroughput,
FeatureNoSVEFPLD1R]>;
def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2",
@@ -590,7 +579,6 @@ def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
- FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive,
FeatureDisableLatencySchedHeuristic]>;
@@ -602,7 +590,6 @@ def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3
FeatureFuseAdrpAdd,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
@@ -613,7 +600,6 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover
FeatureFuseAdrpAdd,
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
- FeatureHalfSVEFCmpThroughput,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive]>;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 2cbd447f7c2d3..3ec45178af68e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4365,6 +4365,34 @@ AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
return 1;
}
+/// Check whether Opcode1 has less throughput according to the scheduling
+/// model than Opcode2.
+bool AArch64TTIImpl::hasLessThroughputFromSchedulingModel(
+ unsigned Opcode1, unsigned Opcode2) const {
+ const MCSchedModel &Sched = ST->getSchedModel();
+ const TargetInstrInfo *TII = ST->getInstrInfo();
+ if (!Sched.hasInstrSchedModel())
+ return false;
+
+ auto ResolveVariant = [&](unsigned Opcode) {
+ unsigned SCIdx = TII->get(Opcode).getSchedClass();
+ while (SCIdx && Sched.getSchedClassDesc(SCIdx)->isVariant())
+ SCIdx = ST->resolveVariantSchedClass(SCIdx, nullptr, TII,
+ Sched.getProcessorID());
+ assert(SCIdx);
+ const MCSchedClassDesc &SCDesc = Sched.SchedClassTable[SCIdx];
+ const MCWriteProcResEntry *B = ST->getWriteProcResBegin(&SCDesc);
+ const MCWriteProcResEntry *E = ST->getWriteProcResEnd(&SCDesc);
+ unsigned Min = Sched.IssueWidth;
+ for (const MCWriteProcResEntry *I = B; I != E; I++)
+ Min = std::min(Min, Sched.getProcResource(I->ProcResourceIdx)->NumUnits /
+ (I->ReleaseAtCycle - I->AcquireAtCycle));
+ return Min;
+ };
+
+ return ResolveVariant(Opcode1) < ResolveVariant(Opcode2);
+}
+
InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
@@ -4463,7 +4491,9 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
Factor = 3; // fcmxx+fcmyy+or
if (isa<ScalableVectorType>(ValTy) &&
- CostKind == TTI::TCK_RecipThroughput && ST->hasHalfSVEFCmpThroughput())
+ CostKind == TTI::TCK_RecipThroughput &&
+ hasLessThroughputFromSchedulingModel(AArch64::FCMEQ_PPzZZ_S,
+ AArch64::FCMEQv4f32))
Factor *= 2;
return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 9c96fdd427814..fe36e43c616c1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -174,6 +174,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool prefersVectorizedAddressing() const override;
+ /// Check whether Opcode1 has less throughput according to the scheduling
+ /// model than Opcode2.
+ bool hasLessThroughputFromSchedulingModel(unsigned Opcode1,
+ unsigned Opcode2) const;
+
InstructionCost
getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 58f2af73bd04c..11bb4d234f3f3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -903,30 +903,23 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
; TFNONE-NEXT: [[ENTRY:.*]]:
; TFNONE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; TFNONE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFNONE: [[VECTOR_BODY]]:
; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFNONE-NEXT: [[TMP7:%.*]] = load double, ptr [[P2]], align 8
-; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP7]], i64 0
-; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
-; TFNONE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true))
-; TFNONE-NEXT: [[TMP9:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP8]], zeroinitializer
-; TFNONE-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
-; TFNONE-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; TFNONE-NEXT: [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 2
-; TFNONE-NEXT: [[TMP13:%.*]] = sub i32 [[TMP12]], 1
-; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i32 [[TMP13]]
+; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
+; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFNONE-NEXT: [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])
+; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
+; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 1
; TFNONE-NEXT: store double [[TMP14]], ptr [[P]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TFNONE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
More information about the llvm-commits
mailing list