[llvm] [AArch64] Improve cost model for legal subvec insert/extract (PR #81135)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 19 07:07:15 PST 2024
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/81135
>From c08072d888ae45f1e33fc180711c0f5f604a581b Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 7 Feb 2024 15:27:41 +0000
Subject: [PATCH 1/2] [AArch64] Don't model legal subvector insert/extract as
scalarization
---
.../AArch64/AArch64TargetTransformInfo.cpp | 42 +++++++++++++++++++
.../CostModel/AArch64/sve-intrinsics.ll | 8 ++--
.../AArch64/scalable-vec-ins-ext.ll | 32 +++++++-------
3 files changed, 64 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 6655931181c2d5..d3e87ab7882d13 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -568,6 +568,48 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
}
return Cost;
}
+ case Intrinsic::vector_extract: {
+ // If both the vector argument and the return type are legal types and the
+ // index is 0, then this should be a no-op or simple operation; return a
+ // relatively low cost.
+
+ // If arguments aren't actually supplied, then we cannot determine the
+ // value of the index.
+ if (ICA.getArgs().size() < 2)
+ break;
+ LLVMContext &C = RetTy->getContext();
+ EVT MRTy = getTLI()->getValueType(DL, RetTy);
+ EVT MPTy = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
+ TargetLoweringBase::LegalizeKind RLK = getTLI()->getTypeConversion(C, MRTy);
+ TargetLoweringBase::LegalizeKind PLK = getTLI()->getTypeConversion(C, MPTy);
+ const ConstantInt *Idx = dyn_cast<ConstantInt>(ICA.getArgs()[1]);
+ if (RLK.first == TargetLoweringBase::TypeLegal &&
+ PLK.first == TargetLoweringBase::TypeLegal && Idx &&
+ Idx->getZExtValue() == 0)
+ return InstructionCost(1);
+ break;
+ }
+ case Intrinsic::vector_insert: {
+ // If both the vector and subvector arguments are legal types and the index
+ // is 0, then this should be a no-op or simple operation; return a
+ // relatively low cost.
+
+ // If arguments aren't actually supplied, then we cannot determine the
+ // value of the index.
+ if (ICA.getArgs().size() < 3)
+ break;
+ LLVMContext &C = RetTy->getContext();
+ EVT MTy0 = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
+ EVT MTy1 = getTLI()->getValueType(DL, ICA.getArgTypes()[1]);
+ TargetLoweringBase::LegalizeKind LK0 = getTLI()->getTypeConversion(C, MTy0);
+ TargetLoweringBase::LegalizeKind LK1 = getTLI()->getTypeConversion(C, MTy1);
+ const ConstantInt *Idx = dyn_cast<ConstantInt>(ICA.getArgs()[2]);
+ if (LK0.first == TargetLoweringBase::TypeLegal &&
+ LK1.first == TargetLoweringBase::TypeLegal && Idx &&
+ Idx->getZExtValue() == 0)
+ return InstructionCost(1);
+ break;
+ }
case Intrinsic::bitreverse: {
static const CostTblEntry BitreverseTbl[] = {
{Intrinsic::bitreverse, MVT::i32, 1},
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 025382878d9931..9eac3dae26bab2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -32,8 +32,8 @@ declare <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x
define void @vector_insert_extract_idxzero_128b() #1 {
; CHECK-LABEL: 'vector_insert_extract_idxzero_128b'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
@@ -61,8 +61,8 @@ declare <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1>
define void @vector_insert_extract_idxzero_256b() #2 {
; CHECK-LABEL: 'vector_insert_extract_idxzero_256b'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
; CHECK-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll b/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll
index 1d5bc84c276357..cf670f5f4f2753 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll
@@ -12,26 +12,30 @@ define void @test_ins_ext_cost(ptr readonly %a, ptr readonly %b, ptr readonly %c
; CHECK-LABEL: define void @test_ins_ext_cost(
; CHECK-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], ptr noalias [[D:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[EXIT_COND:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[FOR_BODY]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 1, [[FOR_BODY]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds <8 x float>, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LOAD_A:%.*]] = load <8 x float>, ptr [[GEP_A]], align 16
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds <8 x float>, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT: [[LOAD_B:%.*]] = load <8 x float>, ptr [[GEP_B]], align 16
-; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds <8 x float>, ptr [[C]], i64 [[IV]]
-; CHECK-NEXT: [[LOAD_C:%.*]] = load <8 x float>, ptr [[GEP_C]], align 16
+; CHECK-NEXT: [[LOAD_A:%.*]] = load <8 x float>, ptr [[A]], align 16
+; CHECK-NEXT: [[LOAD_B:%.*]] = load <8 x float>, ptr [[B]], align 16
+; CHECK-NEXT: [[LOAD_C:%.*]] = load <8 x float>, ptr [[C]], align 16
; CHECK-NEXT: [[CAST_SCALABLE_B:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_B]], i64 0)
; CHECK-NEXT: [[CAST_SCALABLE_C:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_C]], i64 0)
; CHECK-NEXT: [[ADD:%.*]] = fadd <vscale x 4 x float> [[CAST_SCALABLE_B]], [[CAST_SCALABLE_C]]
; CHECK-NEXT: [[CAST_SCALABLE_A:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_A]], i64 0)
; CHECK-NEXT: [[MUL:%.*]] = fmul <vscale x 4 x float> [[CAST_SCALABLE_A]], [[ADD]]
; CHECK-NEXT: [[CAST_FIXED_D:%.*]] = tail call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[MUL]], i64 0)
-; CHECK-NEXT: [[GEP_D:%.*]] = getelementptr inbounds <8 x float>, ptr [[D]], i64 0, i64 [[IV]]
-; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D]], ptr [[GEP_D]], align 16
-; CHECK-NEXT: br i1 [[EXIT_COND]], label [[FOR_BODY]], label [[EXIT:%.*]]
-; CHECK: exit:
+; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D]], ptr [[D]], align 16
+; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[A]], i64 1
+; CHECK-NEXT: [[LOAD_A_1:%.*]] = load <8 x float>, ptr [[GEP_A_1]], align 16
+; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[B]], i64 1
+; CHECK-NEXT: [[LOAD_B_1:%.*]] = load <8 x float>, ptr [[GEP_B_1]], align 16
+; CHECK-NEXT: [[GEP_C_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[C]], i64 1
+; CHECK-NEXT: [[LOAD_C_1:%.*]] = load <8 x float>, ptr [[GEP_C_1]], align 16
+; CHECK-NEXT: [[CAST_SCALABLE_B_1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_B_1]], i64 0)
+; CHECK-NEXT: [[CAST_SCALABLE_C_1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_C_1]], i64 0)
+; CHECK-NEXT: [[ADD_1:%.*]] = fadd <vscale x 4 x float> [[CAST_SCALABLE_B_1]], [[CAST_SCALABLE_C_1]]
+; CHECK-NEXT: [[CAST_SCALABLE_A_1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[LOAD_A_1]], i64 0)
+; CHECK-NEXT: [[MUL_1:%.*]] = fmul <vscale x 4 x float> [[CAST_SCALABLE_A_1]], [[ADD_1]]
+; CHECK-NEXT: [[CAST_FIXED_D_1:%.*]] = tail call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[MUL_1]], i64 0)
+; CHECK-NEXT: [[GEP_D_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[D]], i64 0, i64 1
+; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D_1]], ptr [[GEP_D_1]], align 16
; CHECK-NEXT: ret void
;
entry:
>From 9b6d7d65f20cbdbbafee57f5506336ab8940c14c Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Mon, 19 Feb 2024 14:45:47 +0000
Subject: [PATCH 2/2] Skip predicate types, improve arg count checks
---
.../Target/AArch64/AArch64TargetTransformInfo.cpp | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index d3e87ab7882d13..e1b08c42eb3763 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -574,8 +574,9 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// relatively low cost.
// If arguments aren't actually supplied, then we cannot determine the
- // value of the index.
- if (ICA.getArgs().size() < 2)
+ // value of the index. We also want to skip this on predicate types.
+ if (ICA.getArgs().size() != 2 ||
+ ICA.getReturnType()->getScalarType()->isIntegerTy(1))
break;
LLVMContext &C = RetTy->getContext();
EVT MRTy = getTLI()->getValueType(DL, RetTy);
@@ -586,7 +587,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
if (RLK.first == TargetLoweringBase::TypeLegal &&
PLK.first == TargetLoweringBase::TypeLegal && Idx &&
Idx->getZExtValue() == 0)
- return InstructionCost(1);
+ return TTI::TCC_Basic;
break;
}
case Intrinsic::vector_insert: {
@@ -595,8 +596,9 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// relatively low cost.
// If arguments aren't actually supplied, then we cannot determine the
- // value of the index.
- if (ICA.getArgs().size() < 3)
+ // value of the index. We also want to skip this on predicate types.
+ if (ICA.getArgs().size() != 3 ||
+ ICA.getReturnType()->getScalarType()->isIntegerTy(1))
break;
LLVMContext &C = RetTy->getContext();
EVT MTy0 = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
@@ -607,7 +609,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
if (LK0.first == TargetLoweringBase::TypeLegal &&
LK1.first == TargetLoweringBase::TypeLegal && Idx &&
Idx->getZExtValue() == 0)
- return InstructionCost(1);
+ return TTI::TCC_Basic;
break;
}
case Intrinsic::bitreverse: {
More information about the llvm-commits
mailing list