[llvm] [CostModel][X86] getShuffleCosts - convert all shuffle cost tables to be CostKind compatible. NFC. (PR #124753)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 28 06:22:29 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Simon Pilgrim (RKSimon)
<details>
<summary>Changes</summary>
No change in actual costs yet, but split the costs per cost kind to make it easier to tweak the numbers in future patches.
---
Patch is 41.22 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/124753.diff
1 Files Affected:
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.cpp (+334-329)
``````````diff
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 22dccaf061e1fd..82523bb6557adc 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1673,40 +1673,41 @@ InstructionCost X86TTIImpl::getShuffleCost(
EVT VT = TLI->getValueType(DL, BaseTp);
if (VT.isSimple() && VT.isVector() && VT.getSizeInBits() < 128 &&
!ST->hasSSSE3()) {
- static const CostTblEntry SSE2SubVectorShuffleTbl[] = {
- {TTI::SK_Broadcast, MVT::v4i16, 1}, // pshuflw
- {TTI::SK_Broadcast, MVT::v2i16, 1}, // pshuflw
- {TTI::SK_Broadcast, MVT::v8i8, 2}, // punpck/pshuflw
- {TTI::SK_Broadcast, MVT::v4i8, 2}, // punpck/pshuflw
- {TTI::SK_Broadcast, MVT::v2i8, 1}, // punpck
-
- {TTI::SK_Reverse, MVT::v4i16, 1}, // pshuflw
- {TTI::SK_Reverse, MVT::v2i16, 1}, // pshuflw
- {TTI::SK_Reverse, MVT::v4i8, 3}, // punpck/pshuflw/packus
- {TTI::SK_Reverse, MVT::v2i8, 1}, // punpck
-
- {TTI::SK_Splice, MVT::v4i16, 2}, // punpck+psrldq
- {TTI::SK_Splice, MVT::v2i16, 2}, // punpck+psrldq
- {TTI::SK_Splice, MVT::v4i8, 2}, // punpck+psrldq
- {TTI::SK_Splice, MVT::v2i8, 2}, // punpck+psrldq
-
- {TTI::SK_PermuteTwoSrc, MVT::v4i16, 2}, // punpck/pshuflw
- {TTI::SK_PermuteTwoSrc, MVT::v2i16, 2}, // punpck/pshuflw
- {TTI::SK_PermuteTwoSrc, MVT::v8i8, 7}, // punpck/pshuflw
- {TTI::SK_PermuteTwoSrc, MVT::v4i8, 4}, // punpck/pshuflw
- {TTI::SK_PermuteTwoSrc, MVT::v2i8, 2}, // punpck
-
- {TTI::SK_PermuteSingleSrc, MVT::v4i16, 1}, // pshuflw
- {TTI::SK_PermuteSingleSrc, MVT::v2i16, 1}, // pshuflw
- {TTI::SK_PermuteSingleSrc, MVT::v8i8, 5}, // punpck/pshuflw
- {TTI::SK_PermuteSingleSrc, MVT::v4i8, 3}, // punpck/pshuflw
- {TTI::SK_PermuteSingleSrc, MVT::v2i8, 1}, // punpck
+ static const CostKindTblEntry SSE2SubVectorShuffleTbl[] = {
+ {TTI::SK_Broadcast, MVT::v4i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_Broadcast, MVT::v2i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_Broadcast, MVT::v8i8, {2,2,2,2}}, // punpck/pshuflw
+ {TTI::SK_Broadcast, MVT::v4i8, {2,2,2,2}}, // punpck/pshuflw
+ {TTI::SK_Broadcast, MVT::v2i8, {1,1,1,1}}, // punpck
+
+ {TTI::SK_Reverse, MVT::v4i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_Reverse, MVT::v2i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_Reverse, MVT::v4i8, {3,3,3,3}}, // punpck/pshuflw/packus
+ {TTI::SK_Reverse, MVT::v2i8, {1,1,1,1}}, // punpck
+
+ {TTI::SK_Splice, MVT::v4i16, {2,2,2,2}}, // punpck+psrldq
+ {TTI::SK_Splice, MVT::v2i16, {2,2,2,2}}, // punpck+psrldq
+ {TTI::SK_Splice, MVT::v4i8, {2,2,2,2}}, // punpck+psrldq
+ {TTI::SK_Splice, MVT::v2i8, {2,2,2,2}}, // punpck+psrldq
+
+ {TTI::SK_PermuteTwoSrc, MVT::v4i16, {2,2,2,2}}, // punpck/pshuflw
+ {TTI::SK_PermuteTwoSrc, MVT::v2i16, {2,2,2,2}}, // punpck/pshuflw
+ {TTI::SK_PermuteTwoSrc, MVT::v8i8, {7,7,7,7}}, // punpck/pshuflw
+ {TTI::SK_PermuteTwoSrc, MVT::v4i8, {4,4,4,4}}, // punpck/pshuflw
+ {TTI::SK_PermuteTwoSrc, MVT::v2i8, {2,2,2,2}}, // punpck
+
+ {TTI::SK_PermuteSingleSrc, MVT::v4i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_PermuteSingleSrc, MVT::v2i16, {1,1,1,1}}, // pshuflw
+ {TTI::SK_PermuteSingleSrc, MVT::v8i8, {5,5,5,5}}, // punpck/pshuflw
+ {TTI::SK_PermuteSingleSrc, MVT::v4i8, {3,3,3,3}}, // punpck/pshuflw
+ {TTI::SK_PermuteSingleSrc, MVT::v2i8, {1,1,1,1}}, // punpck
};
if (ST->hasSSE2())
if (const auto *Entry =
CostTableLookup(SSE2SubVectorShuffleTbl, Kind, VT.getSimpleVT()))
- return Entry->Cost;
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
}
// We are going to permute multiple sources and the result will be in multiple
@@ -1803,57 +1804,57 @@ InstructionCost X86TTIImpl::getShuffleCost(
if (LT.first == 1 && IsInLaneShuffle && IsSingleElementMask)
return TTI::TCC_Basic;
- static const CostTblEntry AVX512VBMIShuffleTbl[] = {
- {TTI::SK_Reverse, MVT::v64i8, 1}, // vpermb
- {TTI::SK_Reverse, MVT::v32i8, 1}, // vpermb
-
- {TTI::SK_PermuteSingleSrc, MVT::v64i8, 1}, // vpermb
- {TTI::SK_PermuteSingleSrc, MVT::v32i8, 1}, // vpermb
-
- {TTI::SK_PermuteTwoSrc, MVT::v64i8, 2}, // vpermt2b
- {TTI::SK_PermuteTwoSrc, MVT::v32i8, 2}, // vpermt2b
- {TTI::SK_PermuteTwoSrc, MVT::v16i8, 2} // vpermt2b
+ static const CostKindTblEntry AVX512VBMIShuffleTbl[] = {
+ { TTI::SK_Reverse, MVT::v64i8, { 1, 1, 1, 1 } }, // vpermb
+ { TTI::SK_Reverse, MVT::v32i8, { 1, 1, 1, 1 } }, // vpermb
+ { TTI::SK_PermuteSingleSrc, MVT::v64i8, { 1, 1, 1, 1 } }, // vpermb
+ { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 1, 1, 1, 1 } }, // vpermb
+ { TTI::SK_PermuteTwoSrc, MVT::v64i8, { 2, 2, 2, 2 } }, // vpermt2b
+ { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 2, 2, 2, 2 } }, // vpermt2b
+ { TTI::SK_PermuteTwoSrc, MVT::v16i8, { 2, 2, 2, 2 } } // vpermt2b
};
if (ST->hasVBMI())
if (const auto *Entry =
CostTableLookup(AVX512VBMIShuffleTbl, Kind, LT.second))
- return LT.first * Entry->Cost;
-
- static const CostTblEntry AVX512BWShuffleTbl[] = {
- {TTI::SK_Broadcast, MVT::v32i16, 1}, // vpbroadcastw
- {TTI::SK_Broadcast, MVT::v32f16, 1}, // vpbroadcastw
- {TTI::SK_Broadcast, MVT::v64i8, 1}, // vpbroadcastb
-
- {TTI::SK_Reverse, MVT::v32i16, 2}, // vpermw
- {TTI::SK_Reverse, MVT::v32f16, 2}, // vpermw
- {TTI::SK_Reverse, MVT::v16i16, 2}, // vpermw
- {TTI::SK_Reverse, MVT::v64i8, 2}, // pshufb + vshufi64x2
-
- {TTI::SK_PermuteSingleSrc, MVT::v32i16, 2}, // vpermw
- {TTI::SK_PermuteSingleSrc, MVT::v32f16, 2}, // vpermw
- {TTI::SK_PermuteSingleSrc, MVT::v16i16, 2}, // vpermw
- {TTI::SK_PermuteSingleSrc, MVT::v16f16, 2}, // vpermw
- {TTI::SK_PermuteSingleSrc, MVT::v64i8, 8}, // extend to v32i16
-
- {TTI::SK_PermuteTwoSrc, MVT::v32i16, 2}, // vpermt2w
- {TTI::SK_PermuteTwoSrc, MVT::v32f16, 2}, // vpermt2w
- {TTI::SK_PermuteTwoSrc, MVT::v16i16, 2}, // vpermt2w
- {TTI::SK_PermuteTwoSrc, MVT::v8i16, 2}, // vpermt2w
- {TTI::SK_PermuteTwoSrc, MVT::v64i8, 19}, // 6 * v32i8 + 1
-
- {TTI::SK_Select, MVT::v32i16, 1}, // vblendmw
- {TTI::SK_Select, MVT::v64i8, 1}, // vblendmb
-
- {TTI::SK_Splice, MVT::v32i16, 2}, // vshufi64x2 + palignr
- {TTI::SK_Splice, MVT::v32f16, 2}, // vshufi64x2 + palignr
- {TTI::SK_Splice, MVT::v64i8, 2}, // vshufi64x2 + palignr
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX512BWShuffleTbl[] = {
+ { TTI::SK_Broadcast, MVT::v32i16, { 1, 1, 1, 1 } }, // vpbroadcastw
+ { TTI::SK_Broadcast, MVT::v32f16, { 1, 1, 1, 1 } }, // vpbroadcastw
+ { TTI::SK_Broadcast, MVT::v64i8, { 1, 1, 1, 1 } }, // vpbroadcastb
+
+ { TTI::SK_Reverse, MVT::v32i16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_Reverse, MVT::v32f16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_Reverse, MVT::v16i16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_Reverse, MVT::v64i8, { 2, 2, 2, 2 } }, // pshufb + vshufi64x2
+
+ { TTI::SK_PermuteSingleSrc, MVT::v32i16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_PermuteSingleSrc, MVT::v32f16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 2, 2, 2, 2 } }, // vpermw
+ { TTI::SK_PermuteSingleSrc, MVT::v64i8, { 8, 8, 8, 8 } }, // extend to v32i16
+
+ { TTI::SK_PermuteTwoSrc, MVT::v32i16,{ 2, 2, 2, 2 } }, // vpermt2w
+ { TTI::SK_PermuteTwoSrc, MVT::v32f16,{ 2, 2, 2, 2 } }, // vpermt2w
+ { TTI::SK_PermuteTwoSrc, MVT::v16i16,{ 2, 2, 2, 2 } }, // vpermt2w
+ { TTI::SK_PermuteTwoSrc, MVT::v8i16, { 2, 2, 2, 2 } }, // vpermt2w
+ { TTI::SK_PermuteTwoSrc, MVT::v64i8, { 19, 19, 19, 19 } }, // 6 * v32i8 + 1
+
+ { TTI::SK_Select, MVT::v32i16, { 1, 1, 1, 1 } }, // vblendmw
+ { TTI::SK_Select, MVT::v64i8, { 1, 1, 1, 1 } }, // vblendmb
+
+ { TTI::SK_Splice, MVT::v32i16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
+ { TTI::SK_Splice, MVT::v32f16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
+ { TTI::SK_Splice, MVT::v64i8, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
};
if (ST->hasBWI())
if (const auto *Entry =
CostTableLookup(AVX512BWShuffleTbl, Kind, LT.second))
- return LT.first * Entry->Cost;
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
static const CostKindTblEntry AVX512ShuffleTbl[] = {
{TTI::SK_Broadcast, MVT::v8f64, { 1, 1, 1, 1 } }, // vbroadcastsd
@@ -1934,286 +1935,288 @@ InstructionCost X86TTIImpl::getShuffleCost(
if (auto KindCost = Entry->Cost[CostKind])
return LT.first * *KindCost;
- static const CostTblEntry AVX2InLaneShuffleTbl[] = {
- {TTI::SK_PermuteSingleSrc, MVT::v16i16, 1}, // vpshufb
- {TTI::SK_PermuteSingleSrc, MVT::v16f16, 1}, // vpshufb
- {TTI::SK_PermuteSingleSrc, MVT::v32i8, 1}, // vpshufb
-
- {TTI::SK_PermuteTwoSrc, MVT::v4f64, 2}, // 2*vshufpd + vblendpd
- {TTI::SK_PermuteTwoSrc, MVT::v8f32, 2}, // 2*vshufps + vblendps
- {TTI::SK_PermuteTwoSrc, MVT::v4i64, 2}, // 2*vpshufd + vpblendd
- {TTI::SK_PermuteTwoSrc, MVT::v8i32, 2}, // 2*vpshufd + vpblendd
- {TTI::SK_PermuteTwoSrc, MVT::v16i16, 2}, // 2*vpshufb + vpor
- {TTI::SK_PermuteTwoSrc, MVT::v16f16, 2}, // 2*vpshufb + vpor
- {TTI::SK_PermuteTwoSrc, MVT::v32i8, 2}, // 2*vpshufb + vpor
+ static const CostKindTblEntry AVX2InLaneShuffleTbl[] = {
+ { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 1, 1, 1, 1 } }, // vpshufb
+ { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 1, 1, 1, 1 } }, // vpshufb
+ { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 1, 1, 1, 1 } }, // vpshufb
+
+ { TTI::SK_PermuteTwoSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // 2*vshufpd + vblendpd
+ { TTI::SK_PermuteTwoSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // 2*vshufps + vblendps
+ { TTI::SK_PermuteTwoSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
+ { TTI::SK_PermuteTwoSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
+ { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
+ { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
+ { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
};
if (IsInLaneShuffle && ST->hasAVX2())
if (const auto *Entry =
CostTableLookup(AVX2InLaneShuffleTbl, Kind, LT.second))
- return LT.first * Entry->Cost;
-
- static const CostTblEntry AVX2ShuffleTbl[] = {
- {TTI::SK_Broadcast, MVT::v4f64, 1}, // vbroadcastpd
- {TTI::SK_Broadcast, MVT::v8f32, 1}, // vbroadcastps
- {TTI::SK_Broadcast, MVT::v4i64, 1}, // vpbroadcastq
- {TTI::SK_Broadcast, MVT::v8i32, 1}, // vpbroadcastd
- {TTI::SK_Broadcast, MVT::v16i16, 1}, // vpbroadcastw
- {TTI::SK_Broadcast, MVT::v16f16, 1}, // vpbroadcastw
- {TTI::SK_Broadcast, MVT::v32i8, 1}, // vpbroadcastb
-
- {TTI::SK_Reverse, MVT::v4f64, 1}, // vpermpd
- {TTI::SK_Reverse, MVT::v8f32, 1}, // vpermps
- {TTI::SK_Reverse, MVT::v4i64, 1}, // vpermq
- {TTI::SK_Reverse, MVT::v8i32, 1}, // vpermd
- {TTI::SK_Reverse, MVT::v16i16, 2}, // vperm2i128 + pshufb
- {TTI::SK_Reverse, MVT::v16f16, 2}, // vperm2i128 + pshufb
- {TTI::SK_Reverse, MVT::v32i8, 2}, // vperm2i128 + pshufb
-
- {TTI::SK_Select, MVT::v16i16, 1}, // vpblendvb
- {TTI::SK_Select, MVT::v16f16, 1}, // vpblendvb
- {TTI::SK_Select, MVT::v32i8, 1}, // vpblendvb
-
- {TTI::SK_Splice, MVT::v8i32, 2}, // vperm2i128 + vpalignr
- {TTI::SK_Splice, MVT::v8f32, 2}, // vperm2i128 + vpalignr
- {TTI::SK_Splice, MVT::v16i16, 2}, // vperm2i128 + vpalignr
- {TTI::SK_Splice, MVT::v16f16, 2}, // vperm2i128 + vpalignr
- {TTI::SK_Splice, MVT::v32i8, 2}, // vperm2i128 + vpalignr
-
- {TTI::SK_PermuteSingleSrc, MVT::v4f64, 1}, // vpermpd
- {TTI::SK_PermuteSingleSrc, MVT::v8f32, 1}, // vpermps
- {TTI::SK_PermuteSingleSrc, MVT::v4i64, 1}, // vpermq
- {TTI::SK_PermuteSingleSrc, MVT::v8i32, 1}, // vpermd
- {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vperm2i128 + 2*vpshufb
- // + vpblendvb
- {TTI::SK_PermuteSingleSrc, MVT::v16f16, 4}, // vperm2i128 + 2*vpshufb
- // + vpblendvb
- {TTI::SK_PermuteSingleSrc, MVT::v32i8, 4}, // vperm2i128 + 2*vpshufb
- // + vpblendvb
-
- {TTI::SK_PermuteTwoSrc, MVT::v4f64, 3}, // 2*vpermpd + vblendpd
- {TTI::SK_PermuteTwoSrc, MVT::v8f32, 3}, // 2*vpermps + vblendps
- {TTI::SK_PermuteTwoSrc, MVT::v4i64, 3}, // 2*vpermq + vpblendd
- {TTI::SK_PermuteTwoSrc, MVT::v8i32, 3}, // 2*vpermd + vpblendd
- {TTI::SK_PermuteTwoSrc, MVT::v16i16, 7}, // 2*vperm2i128 + 4*vpshufb
- // + vpblendvb
- {TTI::SK_PermuteTwoSrc, MVT::v16f16, 7}, // 2*vperm2i128 + 4*vpshufb
- // + vpblendvb
- {TTI::SK_PermuteTwoSrc, MVT::v32i8, 7}, // 2*vperm2i128 + 4*vpshufb
- // + vpblendvb
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX2ShuffleTbl[] = {
+ { TTI::SK_Broadcast, MVT::v4f64, { 1, 1, 1, 1 } }, // vbroadcastpd
+ { TTI::SK_Broadcast, MVT::v8f32, { 1, 1, 1, 1 } }, // vbroadcastps
+ { TTI::SK_Broadcast, MVT::v4i64, { 1, 1, 1, 1 } }, // vpbroadcastq
+ { TTI::SK_Broadcast, MVT::v8i32, { 1, 1, 1, 1 } }, // vpbroadcastd
+ { TTI::SK_Broadcast, MVT::v16i16, { 1, 1, 1, 1 } }, // vpbroadcastw
+ { TTI::SK_Broadcast, MVT::v16f16, { 1, 1, 1, 1 } }, // vpbroadcastw
+ { TTI::SK_Broadcast, MVT::v32i8, { 1, 1, 1, 1 } }, // vpbroadcastb
+
+ { TTI::SK_Reverse, MVT::v4f64, { 1, 1, 1, 1 } }, // vpermpd
+ { TTI::SK_Reverse, MVT::v8f32, { 1, 1, 1, 1 } }, // vpermps
+ { TTI::SK_Reverse, MVT::v4i64, { 1, 1, 1, 1 } }, // vpermq
+ { TTI::SK_Reverse, MVT::v8i32, { 1, 1, 1, 1 } }, // vpermd
+ { TTI::SK_Reverse, MVT::v16i16, { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+ { TTI::SK_Reverse, MVT::v16f16, { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+ { TTI::SK_Reverse, MVT::v32i8, { 2, 2, 2, 2 } }, // vperm2i128 + pshufb
+
+ { TTI::SK_Select, MVT::v16i16, { 1, 1, 1, 1 } }, // vpblendvb
+ { TTI::SK_Select, MVT::v16f16, { 1, 1, 1, 1 } }, // vpblendvb
+ { TTI::SK_Select, MVT::v32i8, { 1, 1, 1, 1 } }, // vpblendvb
+
+ { TTI::SK_Splice, MVT::v8i32, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+ { TTI::SK_Splice, MVT::v8f32, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+ { TTI::SK_Splice, MVT::v16i16, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+ { TTI::SK_Splice, MVT::v16f16, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+ { TTI::SK_Splice, MVT::v32i8, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
+
+ { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 1, 1, 1, 1 } }, // vpermpd
+ { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 1, 1, 1, 1 } }, // vpermps
+ { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 1, 1, 1, 1 } }, // vpermq
+ { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 1, 1, 1, 1 } }, // vpermd
+ { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 4, 4, 4, 4 } },
+ { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 4, 4, 4, 4 } },
+ { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } },
+
+ { TTI::SK_PermuteTwoSrc, MVT::v4f64, { 3, 3, 3, 3 } }, // 2*vpermpd + vblendpd
+ { TTI::SK_PermuteTwoSrc, MVT::v8f32, { 3, 3, 3, 3 } }, // 2*vpermps + vblendps
+ { TTI::SK_PermuteTwoSrc, MVT::v4i64, { 3, 3, 3, 3 } }, // 2*vpermq + vpblendd
+ { TTI::SK_PermuteTwoSrc, MVT::v8i32, { 3, 3, 3, 3 } }, // 2*vpermd + vpblendd
+ { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 7, 7, 7, 7 } },
+ { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 7, 7, 7, 7 } },
+ { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 7, 7, 7, 7 } },
};
if (ST->hasAVX2())
if (const auto *Entry = CostTableLookup(AVX2ShuffleTbl, Kind, LT.second))
- return LT.first * Entry->Cost;
-
- static const CostTblEntry XOPShuffleTbl[] = {
- {TTI::SK_PermuteSingleSrc, MVT::v4f64, 2}, // vperm2f128 + vpermil2pd
- {TTI::SK_PermuteSingleSrc, MVT::v8f32, 2}, // vperm2f128 + vpermil2ps
- {TTI::SK_PermuteSingleSrc, MVT::v4i64, 2}, // vperm2f128 + vpermil2pd
- {TTI::SK_PermuteSingleSrc, MVT::v8i32, 2}, // vperm2f128 + vpermil2ps
- {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vextractf128 + 2*vpperm
- // + vinsertf128
- {TTI::SK_PermuteSingleSrc, MVT::v32i8, 4}, // vextractf128 + 2*vpperm
- // + vinsertf128
-
- {TTI::SK_PermuteTwoSrc, MVT::v16i16, 9}, // 2*vextractf128 + 6*vpperm
- // + vinsertf128
- {TTI::SK_PermuteTwoSrc, MVT::v8i16, 1}, // vpperm
- {TTI::SK_PermuteTwoSrc, MVT::v32i8, 9}, // 2*vextractf128 + 6*vpperm
- // + vinsertf128
- {TTI::SK_PermuteTwoSrc, MVT::v16i8, 1}, // vpperm
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry XOPShuffleTbl[] = {
+ { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
+ { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
+ { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
+ { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
+ { TTI::SK_PermuteSingleSrc, MVT::v16i16,{ 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
+ // + vinsertf128
+ { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
+ // + vinsertf128
+
+ { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
+ // + vinsertf128
+
+ { TTI::SK_PermuteTwoSrc, MVT::v8i16, { 1, 1, 1, 1 } }, // vpperm
+ { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
+ // + vinsertf128
+ { TTI::SK_PermuteTwoSrc, MVT::v16i8, { 1, 1, 1, 1 } }, // vpperm
};
if (ST->hasXOP())
if (const auto *Entry = CostTableLookup(XOPShuffleTbl, Kind, LT.second))
- return LT.first * Entry->Cost;
-
- static const CostTblEntry AVX1InLaneShuffleTbl[] = {
- {TTI::SK_PermuteSingleSrc, MVT::v4f64, 1}, // vpermilpd
- {TTI::SK_PermuteSingleSrc, MVT::v4i64, 1}, // vpermilpd
- {TTI::SK_PermuteSingleSrc, MVT::v8f32, 1}, // vpermilps
- {TTI::SK_PermuteSingleSrc, MVT::v8i32, 1}, // vpermilps
-
- {TTI::SK_PermuteSingleSrc, MVT::v16i16, 4}, // vextractf128 + 2*pshufb
- // + vpor + vinsertf128
- {TTI::SK_PermuteSingleSrc, MVT::v16f16, 4}, // vextractf128 + 2*pshufb
- // + vpor + vinsertf128
- {TTI::SK_PermuteSingleSrc, MVT::v32i8, 4}, // vextractf128 + 2*pshufb
- // + vpor + vinsertf128
-
- {TTI::SK_PermuteTwoSrc, MVT::v4f64, 2}, // 2*vshufpd + vblendpd
- {TTI::SK_PermuteTwoSrc, MVT::v8f32, 2}, // 2*vshufps + vblendps
- {TTI::SK_PermuteTwoSrc, MVT::v4i64, 2}, // 2*vpermilpd + vblendpd
- {TTI::SK_PermuteTwoSrc, MVT::v8i32, 2}, // 2*vpermilps + vblendps
- {TTI::SK_PermuteTwoSrc, MVT::v16i16, 9}, // 2*vextractf128 + 4*pshufb
- ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/124753
More information about the llvm-commits
mailing list