[llvm-branch-commits] [llvm] [SLP] Drop the context of a vector fmul whose user is not vectorized (PR #228902)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 4 17:18:57 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/228902
>From 2c1d4289e7cdbbd98a69e1324b126090bea52125 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 4 Oct 2026 03:30:00 +0200
Subject: [PATCH 1/2] [SLP] Drop the context of a vector fmul whose user is not
vectorized
Pass the context of a vector fmul only when its user is vectorized or
the target emits it lane by lane. AMDGPU prices an fmul with a contract
fadd user as free, which does not hold for the extracted lanes of a
root. Other operations keep their context for the fast math flags and
the function attributes.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 20 ++++-
.../CodeGen/AMDGPU/slp-coalesced-loads.ll | 30 +++----
.../ordered-reduction-coalesced-loads.ll | 80 +++++++++++++++----
.../AMDGPU/ordered-reduction-fma-fusion.ll | 75 ++++++++---------
4 files changed, 131 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c105f41bc15ab..25953e843be40 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17575,9 +17575,22 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
TTI::OperandValueInfo Op1Info = getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op2Info = getOperandInfo(E->getOperand(OpIdx));
+ // A target may price an fmul by the fadd or fsub user of its context.
+ // The vector fmul keeps that user when this tree vectorizes the user
+ // too or when the target emits the fmul lane by lane. The lanes of a
+ // root are extracted otherwise and their scalar users stay as they are.
+ // Every other operation keeps its context, targets read the fast math
+ // flags and the function attributes from it.
+ const TreeEntry *UserTE = E->UserTreeIndex.UserTE;
+ bool KeepsContext =
+ ShuffleOrOp != Instruction::FMul ||
+ (UserTE && !UserTE->isGather() && !DeletedNodes.contains(UserTE) &&
+ !TransformedToGatherNodes.contains(UserTE)) ||
+ TTI->getMaximumVF(DL->getTypeSizeInBits(ScalarTy), ShuffleOrOp) == 1;
+ Instruction *CxtI =
+ VL0->getOpcode() == ShuffleOrOp && KeepsContext ? VL0 : nullptr;
InstructionCost Cost = TTI->getArithmeticInstrCost(
- ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {},
- VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
+ ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {}, CxtI, TLI);
// N columns need N-1 vector combines; price extra columns
// conservatively, skipping identity-only columns (not combined by
// codegen).
@@ -17592,8 +17605,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// static properties to model.
Cost += TTI->getArithmeticInstrCost(
ShuffleOrOp, VecTy, CostKind, {},
- getOperandInfo(E->getOperand(Idx)), {},
- VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
+ getOperandInfo(E->getOperand(Idx)), {}, CxtI, TLI);
}
}
return Cost + CommonCost;
diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
index 4ac9d6adf090e..4db68adac00aa 100644
--- a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
@@ -284,27 +284,23 @@ define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %ac
; GFX1250-NEXT: global_load_b128 v[6:9], v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x1
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_ashrrev_i32_e32 v0, 16, v10
+; GFX1250-NEXT: v_bfe_i32 v0, v10, 0, 16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_bfe_i32 v2, v10, 0, 16
-; GFX1250-NEXT: v_bfe_i32 v5, v11, 0, 16
+; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 16, v10 :: v_dual_ashrrev_i32 v3, 16, v11
+; GFX1250-NEXT: v_bfe_i32 v2, v11, 0, 16
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v0
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v2
-; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 16, v11
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v2
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v0, v4, v0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[8:9]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: v_fma_f32 v0, v0, v6, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v1, v7
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v3
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v2, v8
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v1, v9
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
entry:
%a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
index f7f2f69c1366e..3a41dc15103c1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
@@ -167,22 +167,70 @@ declare i64 @_Z13get_global_idj(i32 noundef)
; coalesced loads and stay scalar too.
define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
-; CHECK-LABEL: define float @dot_i16_window(
-; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[M0:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; CHECK-NEXT: [[M1:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
-; CHECK-NEXT: [[M2:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
-; CHECK-NEXT: [[M3:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
-; CHECK-NEXT: ret float [[S3]]
+; GFX9-LABEL: define float @dot_i16_window(
+; GFX9-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; GFX9-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; GFX9-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; GFX9-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; GFX9-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; GFX9-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; GFX9-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; GFX9-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; GFX9-NEXT: ret float [[S3]]
+;
+; GFX1030-LABEL: define float @dot_i16_window(
+; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; GFX1030-NEXT: ret float [[S3]]
+;
+; GFX1250-LABEL: define float @dot_i16_window(
+; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX1250-NEXT: [[ENTRY:.*:]]
+; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
+; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
+; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
+; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
+; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
+; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
+; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
+; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
+; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
+; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
+; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
+; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
+; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
+; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
+; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
+; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
+; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
+; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
+; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
+; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
+; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
+; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
+; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX1250-NEXT: ret float [[S3]]
;
entry:
%a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index 6a7a93a44f8c9..a25409530b4bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -8,44 +8,44 @@
define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-LABEL: @conv_contract(
-; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
-; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
-; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
+; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
; CHECK-NEXT: [[IV4:%.*]] = load float, ptr addrspace(1) [[IP4]], align 4
-; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
+; CHECK-NEXT: [[MP1:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
+; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4
+; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV4]], [[MV0]]
+; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1
+; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1
+; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
+; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12
; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4
-; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
-; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
-; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
-; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
-; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20
+; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4
-; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
-; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4
+; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17
+; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19
+; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[IP6]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV4]], i64 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV5]], i64 5
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <16 x float> [[TMP8]], float [[IV14]], i64 14
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[IV15]], i64 15
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV5]], i64 8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV14]], i64 9
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP24]], <16 x float> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP20]], [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP20]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP24]], [[TMP6]]
; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
+; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[ACC14]]
; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
-; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[PROD15]]
+; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC1]], [[PROD15]]
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]]
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
@@ -74,15 +74,22 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
-; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
-; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
+; CHECK-NEXT: [[IP17:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 26
+; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 17
+; CHECK-NEXT: [[IP19:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 28
+; CHECK-NEXT: [[IV19:%.*]] = load float, ptr addrspace(1) [[IP19]], align 4
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
-; CHECK-NEXT: [[TMP38:%.*]] = load <4 x float>, ptr addrspace(1) [[IP16]], align 4
-; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33
+; CHECK-NEXT: [[TMP38:%.*]] = load <2 x float>, ptr addrspace(1) [[IP17]], align 4
+; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP21]], align 4
; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP41:%.*]] = insertelement <8 x float> poison, float [[IV19]], i64 2
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP41]], float [[IV20]], i64 3
; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> [[TMP39]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <8 x float> [[TMP53]], <8 x float> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x float> [[TMP38]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <8 x float> [[TMP54]], <8 x float> [[TMP55]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]]
@@ -99,13 +106,7 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6
; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]]
; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7
-; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
-; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36
-; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4
-; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24
-; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4
-; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]]
-; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[PROD20]]
+; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
; CHECK-NEXT: ret float [[ACC24]]
;
%ip0 = getelementptr inbounds float, ptr addrspace(1) %input, i64 0
>From de4df3e6ded0f8ea0a016e9dc68fd0c5c6d3ca4b Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 4 Oct 2026 22:08:01 +0200
Subject: [PATCH 2/2] Tell the target when the users of a cost context stay
scalar
Add a hint to getArithmeticInstrCost that says whether the users of the
context use the priced operation or see its lanes extracted. SLP passes
it for vector binops, the users stay scalar when the user node is a
gather or is not vectorized.
---
.../llvm/Analysis/TargetTransformInfo.h | 14 ++++++++++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 3 ++-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 10 ++++----
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 ++--
.../AArch64/AArch64TargetTransformInfo.cpp | 5 ++--
.../AArch64/AArch64TargetTransformInfo.h | 5 ++--
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 ++++++----
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 5 ++--
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 3 ++-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 5 ++--
llvm/lib/Target/BPF/BPFTargetTransformInfo.h | 5 ++--
.../Hexagon/HexagonTargetTransformInfo.cpp | 9 +++----
.../Hexagon/HexagonTargetTransformInfo.h | 5 ++--
.../Target/Lanai/LanaiTargetTransformInfo.h | 5 ++--
.../Target/NVPTX/NVPTXTargetTransformInfo.cpp | 3 ++-
.../Target/NVPTX/NVPTXTargetTransformInfo.h | 5 ++--
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 7 +++---
.../Target/PowerPC/PPCTargetTransformInfo.h | 5 ++--
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 13 +++++-----
.../Target/RISCV/RISCVTargetTransformInfo.h | 5 ++--
.../SystemZ/SystemZTargetTransformInfo.cpp | 9 +++----
.../SystemZ/SystemZTargetTransformInfo.h | 5 ++--
.../WebAssemblyTargetTransformInfo.cpp | 3 ++-
.../WebAssemblyTargetTransformInfo.h | 5 ++--
.../lib/Target/X86/X86TargetTransformInfo.cpp | 5 ++--
llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 ++--
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++------------
27 files changed, 108 insertions(+), 76 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index a33e6f62e941e..0f9207827c1e8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1562,6 +1562,16 @@ class TargetTransformInfo {
LLVM_ABI static OperandValueInfo commonOperandInfo(const Value *X,
const Value *Y);
+ /// Represents a hint about the users of the context instruction passed to
+ /// getArithmeticInstrCost. A vectorizer prices the vector form of an
+ /// instruction with that instruction as the context, the hint says whether
+ /// the users of the context use the priced operation as well.
+ enum class ContextUsersHint : uint8_t {
+ Kept, ///< The users of the context use the priced operation.
+ Extracted, ///< The lanes of the priced operation are extracted or
+ ///< shuffled before they reach the users of the context.
+ };
+
/// This is an approximation of reciprocal throughput of a math/logic op.
/// A higher cost indicates less expected throughput.
/// From Agner Fog's guides, reciprocal throughput is "the average number of
@@ -1580,12 +1590,14 @@ class TargetTransformInfo {
/// provide even more information.
/// \p TLibInfo is used to search for platform specific vector library
/// functions for instructions that might be converted to calls (e.g. frem).
+ /// \p UsersHint says whether the users of \p CtxI use the priced operation.
LLVM_ABI InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
- const TargetLibraryInfo *TLibInfo = nullptr) const;
+ const TargetLibraryInfo *TLibInfo = nullptr,
+ ContextUsersHint UsersHint = ContextUsersHint::Kept) const;
/// Returns the cost estimation for alternating opcode pattern that can be
/// lowered to a single instruction on the target. In X86 this is for the
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a19c122c16f20..28d083af2ec38 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -738,7 +738,8 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI = nullptr) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint = TTI::ContextUsersHint::Kept) const {
// Widenable conditions will eventually lower into constants, so some
// operations with them will be trivially optimized away.
auto IsWidenableCondition = [](const Value *V) {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 0d9b361e95ad5..342740726e680 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1094,8 +1094,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override {
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override {
// Check if any of the operands are vector operands.
const TargetLoweringBase *TLI = getTLI();
int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -1103,9 +1104,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
- Opd1Info, Opd2Info,
- Args, CtxI);
+ return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
+ Opd2Info, Args, CtxI, UsersHint);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index af73a615f25fa..7427abe538beb 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1031,7 +1031,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
OperandValueInfo Op1Info, OperandValueInfo Op2Info,
ArrayRef<const Value *> Args, const Instruction *CtxI,
- const TargetLibraryInfo *TLibInfo) const {
+ const TargetLibraryInfo *TLibInfo, ContextUsersHint UsersHint) const {
// Use call cost for frem intructions that have platform specific vector math
// functions, as those will be replaced with calls later by SelectionDAG or
@@ -1046,7 +1046,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
}
InstructionCost Cost = TTIImpl->getArithmeticInstrCost(
- Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI);
+ Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI, UsersHint);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 1c815c439aca0..d6cab29b94ec9 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4929,7 +4929,8 @@ std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// The code-generator is currently not able to handle scalable vectors
// of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
@@ -4948,7 +4949,7 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
ISD == ISD::FDIV || ISD == ISD::FREM || ISD == ISD::FNEG)
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
- Op2Info, Args, CtxI);
+ Op2Info, Args, CtxI, UsersHint);
if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
ISD == ISD::FDIV || ISD == ISD::FREM) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index d090f69c1476a..3af00cbb4614f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -250,8 +250,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost
getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 9662be4530f78..18e4fb4d3771f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -577,7 +577,8 @@ static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty,
InstructionCost GCNTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
@@ -635,8 +636,11 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
case ISD::FMUL:
// Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
// fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
- // fused operation.
- if (CtxI && CtxI->hasOneUse()) {
+ // fused operation. A vector fmul whose lanes are extracted before the
+ // fadd|fsub fuses only when it is emitted lane by lane.
+ if (CtxI && CtxI->hasOneUse() &&
+ (UsersHint == TTI::ContextUsersHint::Kept ||
+ getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1)) {
const auto *FAddSub = dyn_cast<BinaryOperator>(*CtxI->user_begin());
if (FAddSub &&
(FAddSub->getOpcode() == Instruction::FAdd ||
@@ -726,7 +730,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
}
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
// Return true if there's a potential benefit from using v2f16/v2i16
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 593aa5ebc43a2..e00628625af88 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -169,8 +169,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 3ea4ddd10a00a..ce1c7e3d071e2 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1390,7 +1390,8 @@ InstructionCost ARMTTIImpl::getShuffleCost(
InstructionCost ARMTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) {
// Make operations on i1 relatively expensive as this often involves
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 7dacff87fae36..5b2dc2c114464 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -237,8 +237,9 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost getMemoryOpCost(
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 0d5ff2d36aca9..81267a3853f6c 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -62,8 +62,9 @@ class BPFTTIImpl final : public BasicTTIImplBase<BPFTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override {
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
if (ISD == ISD::ADD && CostKind == TTI::TCK_RecipThroughput)
return SCEVCheapExpansionBudget.getValue() + 1;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 263eaa7ebb64f..f4448d89a2215 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -269,11 +269,12 @@ InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
- Op2Info, Args, CtxI);
+ return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+ Args, CtxI, UsersHint);
if (Ty->isVectorTy()) {
if (!isHVXVectorType(Ty) && Ty->isFPOrFPVectorTy())
@@ -283,7 +284,7 @@ InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
return LT.first + FloatFactor * getTypeNumElements(Ty);
}
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index dbc88e40e9d5a..e1ff081fe962f 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -145,8 +145,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 78f8eb62cb789..4d0c2016270a3 100644
--- a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -95,8 +95,9 @@ class LanaiTTIImpl final : public BasicTTIImplBase<LanaiTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override {
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
switch (ISD) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 72ec95abeb509..169fef4f95e8b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -501,7 +501,8 @@ NVPTXTTIImpl::getInstructionCost(const User *U,
InstructionCost NVPTXTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index a9c8964c55eac..4560e0e56ae36 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -126,8 +126,9 @@ class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost
getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 0c14942f839f5..660ce5b3c5799 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -599,7 +599,8 @@ InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode,
InstructionCost PPCTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr);
@@ -608,8 +609,8 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
- Op2Info, Args, CtxI);
+ return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+ Args, CtxI, UsersHint);
// Fallback to the default implementation.
InstructionCost Cost = BaseT::getArithmeticInstrCost(
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 3702283c54340..371dac17a7b8e 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -109,8 +109,9 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 69554ca2b8155..b9ceda9d8548f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -2956,21 +2956,22 @@ RISCVTTIImpl::getCombinedArithmeticInstructionCost(
InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
// Skip if scalar size of Ty is bigger than ELEN.
if (isa<VectorType>(Ty) && Ty->getScalarSizeInBits() > ST->getELen())
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
if (std::optional<InstructionCost> CombinedCost =
getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Op1Info,
@@ -2997,7 +2998,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
return Entry->Cost * LT.first;
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
// f16 with zvfhmin and bf16 will be promoted to f32.
@@ -3089,7 +3090,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
// differentiate them.
return CastCost + ConstantMatCost +
BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 2d179479565b5..bbff43638f6ef 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -280,8 +280,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
bool isElementTypeLegalForScalableVector(Type *Ty) const override {
return TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, Ty));
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 06f84c78a504d..9e4da423c4b4d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -611,12 +611,13 @@ static bool isFoldableRMW(const Instruction *I, Type *Ty) {
InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
- Op2Info, Args, CtxI);
+ return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+ Args, CtxI, UsersHint);
if (CtxI && Ty && !Ty->isVectorTy() && isFoldableRMW(CtxI, Ty))
return TTI::TCC_Free;
// TODO: return a good value for BB-VECTORIZER that includes the
@@ -791,7 +792,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
// Fallback to the default implementation.
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
InstructionCost SystemZTTIImpl::getShuffleCost(
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 8236790a6e271..bce3eafdc130d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -102,8 +102,9 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost getPartialReductionCost(
unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 208f126ab2606..2d895ab6e3c8e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -56,7 +56,8 @@ TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
if (ST->hasSIMD128()) {
static const CostTblEntry ArithCostTbl[]{
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index bfa83a0312a0f..79d1e6857467e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -66,8 +66,9 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e42916fe8b5d5..da2e89e44260d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -255,7 +255,8 @@ unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF,
InstructionCost X86TTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
- ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+ ArrayRef<const Value *> Args, const Instruction *CtxI,
+ TTI::ContextUsersHint UsersHint) const {
// vXi8 multiplications are always promoted to vXi16.
// Sub-128-bit types can be extended/packed more efficiently.
@@ -1933,7 +1934,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
// Fallback to the default implementation.
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
- Args, CtxI);
+ Args, CtxI, UsersHint);
}
InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 41fbcc77304fb..b03ce2a09fd8c 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -70,8 +70,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
- ArrayRef<const Value *> Args = {},
- const Instruction *CtxI = nullptr) const override;
+ ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+ TTI::ContextUsersHint UsersHint =
+ TTI::ContextUsersHint::Kept) const override;
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
const SmallBitVector &OpcodeMask,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 25953e843be40..281221838a277 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17575,22 +17575,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
TTI::OperandValueInfo Op1Info = getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op2Info = getOperandInfo(E->getOperand(OpIdx));
- // A target may price an fmul by the fadd or fsub user of its context.
- // The vector fmul keeps that user when this tree vectorizes the user
- // too or when the target emits the fmul lane by lane. The lanes of a
- // root are extracted otherwise and their scalar users stay as they are.
- // Every other operation keeps its context, targets read the fast math
- // flags and the function attributes from it.
const TreeEntry *UserTE = E->UserTreeIndex.UserTE;
- bool KeepsContext =
- ShuffleOrOp != Instruction::FMul ||
- (UserTE && !UserTE->isGather() && !DeletedNodes.contains(UserTE) &&
- !TransformedToGatherNodes.contains(UserTE)) ||
- TTI->getMaximumVF(DL->getTypeSizeInBits(ScalarTy), ShuffleOrOp) == 1;
- Instruction *CxtI =
- VL0->getOpcode() == ShuffleOrOp && KeepsContext ? VL0 : nullptr;
+ TTI::ContextUsersHint UsersHint =
+ UserTE && !UserTE->isGather() && !DeletedNodes.contains(UserTE) &&
+ !TransformedToGatherNodes.contains(UserTE)
+ ? TTI::ContextUsersHint::Kept
+ : TTI::ContextUsersHint::Extracted;
InstructionCost Cost = TTI->getArithmeticInstrCost(
- ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {}, CxtI, TLI);
+ ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {},
+ VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI, UsersHint);
// N columns need N-1 vector combines; price extra columns
// conservatively, skipping identity-only columns (not combined by
// codegen).
@@ -17605,7 +17598,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// static properties to model.
Cost += TTI->getArithmeticInstrCost(
ShuffleOrOp, VecTy, CostKind, {},
- getOperandInfo(E->getOperand(Idx)), {}, CxtI, TLI);
+ getOperandInfo(E->getOperand(Idx)), {},
+ VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI, UsersHint);
}
}
return Cost + CommonCost;
More information about the llvm-branch-commits
mailing list