[llvm] 36c3b79 - [SLP] Fix canConvertToFMA fmul costing (#216425)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 07:02:22 PDT 2026
Author: Dmitry Sidorov
Date: 2026-08-22T16:02:17+02:00
New Revision: 36c3b79de6442f9b6a9756822d57d0fe7af6f23e
URL: https://github.com/llvm/llvm-project/commit/36c3b79de6442f9b6a9756822d57d0fe7af6f23e
DIFF: https://github.com/llvm/llvm-project/commit/36c3b79de6442f9b6a9756822d57d0fe7af6f23e.diff
LOG: [SLP] Fix canConvertToFMA fmul costing (#216425)
Price the unfused fmul without a context instruction. Targets that model
fma fusion price a contractable fmul as free, which discounted the
scalar side of the comparison too and fmuladd never looked profitable.
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d90c8f5fd336e..6a947cda11751 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14489,16 +14489,40 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
+ // Price both sides of the fmul+fadd pair as not fused. Passing a context
+ // instruction would let targets that model the fusion discount the unfused
+ // side of the comparison as well.
+ auto GetUnfusedFMulCost = [&](Instruction *I) {
+ assert(I->getOpcode() == Instruction::FMul && "Expected an fmul");
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
+ Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
+ };
FastMathFlags FMF;
FMF.set();
+ const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
+ S.isShiftOp() || S.isBitwiseLogicOp();
for (Value *V : VL) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
continue;
- if (!S.isCopyableElement(I))
+ const bool IsCopyable = S.isCopyableElement(I);
+ if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ if (IsCopyable || !IsArithmeticState ||
+ (I->getOpcode() != S.getOpcode() &&
+ I->getOpcode() != S.getAltOpcode())) {
+ FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ continue;
+ }
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ FMulPlusFAddCost += TTI.getArithmeticInstrCost(
+ I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
}
unsigned NumOps = 0;
for (auto [V, Op] : zip(VL, Operands.front())) {
@@ -14515,7 +14539,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ FMulPlusFAddCost += GetUnfusedFMulCost(I);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
index 9723bdd6bb369..9b37c518981e3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
@@ -126,12 +126,42 @@ define void @contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr no
; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
-; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
-; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
-; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
-; THR15-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP0]], [[TMP1]]
-; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
-; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4
+; THR15-NEXT: [[A0:%.*]] = load float, ptr [[AP0]], align 4
+; THR15-NEXT: [[B0:%.*]] = load float, ptr [[BP0]], align 4
+; THR15-NEXT: [[C0:%.*]] = load float, ptr [[CP0]], align 4
+; THR15-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; THR15-NEXT: [[R0:%.*]] = fadd contract float [[M0]], [[C0]]
+; THR15-NEXT: store float [[R0]], ptr [[DP0]], align 4
+; THR15-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; THR15-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; THR15-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; THR15-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; THR15-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; THR15-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; THR15-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; THR15-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; THR15-NEXT: [[R1:%.*]] = fadd contract float [[M1]], [[C1]]
+; THR15-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; THR15-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; THR15-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; THR15-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; THR15-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; THR15-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; THR15-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; THR15-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; THR15-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; THR15-NEXT: [[R2:%.*]] = fadd contract float [[M2]], [[C2]]
+; THR15-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; THR15-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; THR15-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; THR15-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; THR15-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; THR15-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; THR15-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; THR15-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; THR15-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; THR15-NEXT: [[R3:%.*]] = fadd contract float [[M3]], [[C3]]
+; THR15-NEXT: store float [[R3]], ptr [[DP3]], align 4
; THR15-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list