[llvm] [SLP] Fix canConvertToFMA operand selection and fmul costing (PR #216425)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 15:55:27 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Dmitry Sidorov (MrSidims)
<details>
<summary>Changes</summary>
canConvertToFMA only looked for the fmul on operand 0 of the fadd, so the accumulator shape fadd acc, a * b was never recognized. Check both operands for chains that do not allow reassociation.
Also price the unfused fmul without a context instruction. Targets that model fma fusion price a contractable fmul as free, which discounted the scalar side of the comparison too and fmuladd never looked profitable.
---
Full diff: https://github.com/llvm/llvm-project/pull/216425.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+39-10)
- (added) llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll (+130)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53816d49de722..6e5628104f0f9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14382,18 +14382,44 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
- InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
- if (!OpS.valid())
- return InstructionCost::getInvalid();
-
- if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
- return InstructionCost::getInvalid();
- if (!CheckForContractable(Operands.front()))
+ // The fmul may sit on either side of the add/sub. Look past operand 0 only
+ // for chains that do not allow reassociation. A reassociative chain can be
+ // vectorized into a vector fmul feeding a reduction, which is usually
+ // better than the scalar fma chain this check protects.
+ bool AllowReassoc = any_of(VL, [](Value *V) {
+ auto *FPCI = dyn_cast<FPMathOperator>(V);
+ return FPCI && FPCI->getFastMathFlags().allowReassoc();
+ });
+ auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
+ for (unsigned Idx : seq<unsigned>(0, AllowReassoc ? 1 : Operands.size())) {
+ InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
+ if (!CandS.valid() || CandS.isAltShuffle() ||
+ CandS.getOpcode() != Instruction::FMul)
+ continue;
+ if (!CheckForContractable(Operands[Idx]))
+ continue;
+ return Idx;
+ }
+ return std::nullopt;
+ };
+ std::optional<unsigned> FMulIdx = GetFMulOperandIdx();
+ if (!FMulIdx)
return InstructionCost::getInvalid();
+ InstructionsState OpS = getSameOpcode(Operands[*FMulIdx], TLI);
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ // Price the fmul as not fused with its user. Passing a context instruction
+ // would let targets that model the fusion discount the unfused side of the
+ // comparison as well.
+ auto GetUnfusedFMulCost = [&](Instruction *I) {
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(Instruction::FMul, I->getType(), CostKind,
+ Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
+ };
FastMathFlags FMF;
FMF.set();
for (Value *V : VL) {
@@ -14406,7 +14432,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
}
unsigned NumOps = 0;
- for (auto [V, Op] : zip(VL, Operands.front())) {
+ for (auto [V, Op] : zip(VL, Operands[*FMulIdx])) {
if (S.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(Op);
@@ -14420,7 +14446,9 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ FMulPlusFAddCost += I->getOpcode() == Instruction::FMul
+ ? GetUnfusedFMulCost(I)
+ : TTI.getInstructionCost(I, CostKind);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
@@ -15225,7 +15253,8 @@ void BoUpSLP::transformNodes() {
break;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
- TreeEntry *FMulEntry = getOperandEntry(&E, 0);
+ TreeEntry *FMulEntry =
+ getOperandEntry(&E, IsOneUseVectorFMulOperand(LHS) ? 0 : 1);
if (FMulEntry->UserTreeIndex &&
FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
new file mode 100644
index 0000000000000..efa8f5c4dd930
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
+
+; Elementwise d = c + a * b where the fmul is operand 1 of the fadd. Marking
+; a hardcoded operand 0 turned the c load bundle into a CombinedVectorize
+; node and made vectorizeTree abort.
+
+define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @axpy4_contract(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP5]], [[TMP8]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %c0 = load float, ptr %c, align 4
+ %a0 = load float, ptr %a, align 4
+ %b0 = load float, ptr %b, align 4
+ %m0 = fmul contract float %a0, %b0
+ %r0 = fadd contract float %c0, %m0
+ store float %r0, ptr %d, align 4
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %c1 = load float, ptr %cp1, align 4
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %a1 = load float, ptr %ap1, align 4
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %b1 = load float, ptr %bp1, align 4
+ %m1 = fmul contract float %a1, %b1
+ %r1 = fadd contract float %c1, %m1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ store float %r1, ptr %dp1, align 4
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %c2 = load float, ptr %cp2, align 4
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %a2 = load float, ptr %ap2, align 4
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %b2 = load float, ptr %bp2, align 4
+ %m2 = fmul contract float %a2, %b2
+ %r2 = fadd contract float %c2, %m2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ store float %r2, ptr %dp2, align 4
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %c3 = load float, ptr %cp3, align 4
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %a3 = load float, ptr %ap3, align 4
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %b3 = load float, ptr %bp3, align 4
+ %m3 = fmul contract float %a3, %b3
+ %r3 = fadd contract float %c3, %m3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ store float %r3, ptr %dp3, align 4
+ ret void
+}
+
+define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @axpy4_reassoc(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc contract <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc contract <2 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul reassoc contract <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd reassoc contract <2 x float> [[TMP5]], [[TMP8]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %c0 = load float, ptr %c, align 4
+ %a0 = load float, ptr %a, align 4
+ %b0 = load float, ptr %b, align 4
+ %m0 = fmul contract reassoc float %a0, %b0
+ %r0 = fadd contract reassoc float %c0, %m0
+ store float %r0, ptr %d, align 4
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %c1 = load float, ptr %cp1, align 4
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %a1 = load float, ptr %ap1, align 4
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %b1 = load float, ptr %bp1, align 4
+ %m1 = fmul contract reassoc float %a1, %b1
+ %r1 = fadd contract reassoc float %c1, %m1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ store float %r1, ptr %dp1, align 4
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %c2 = load float, ptr %cp2, align 4
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %a2 = load float, ptr %ap2, align 4
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %b2 = load float, ptr %bp2, align 4
+ %m2 = fmul contract reassoc float %a2, %b2
+ %r2 = fadd contract reassoc float %c2, %m2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ store float %r2, ptr %dp2, align 4
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %c3 = load float, ptr %cp3, align 4
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %a3 = load float, ptr %ap3, align 4
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %b3 = load float, ptr %bp3, align 4
+ %m3 = fmul contract reassoc float %a3, %b3
+ %r3 = fadd contract reassoc float %c3, %m3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ store float %r3, ptr %dp3, align 4
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/216425
More information about the llvm-commits
mailing list