[llvm] [SLP]Support copyables in the fmuladd multiplicand (PR #212808)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 08:56:22 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/212808
Allow a non-fmuladd lane V to be modeled as fmuladd(V, 1.0, -0.0),
which equals V for every V (V * 1.0 is exact, and V + -0.0 preserves
signed zeros). The addend modeling is still preferred; the
multiplicand is used as a fallback when the addend operand does not
allow vectorization. The copyable position is kept as an operand
index in InstructionsState, so other operands can be modeled later.
>From 264f240f5e86ae1746cc2622b0ce912de881cb9d Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 29 Jul 2026 08:56:07 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 46 ++++++++++++---
.../X86/fmuladd-copyable-mul-part.ll | 56 ++++++++++---------
2 files changed, 66 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3e4d3490a69aa..210b169c288c5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -607,6 +607,9 @@ class InstructionsState {
Instruction *AltOp = nullptr;
/// Wether the instruction state represents copyable instructions.
bool HasCopyables = false;
+ /// Index of the operand modeling the copyable values: the addend for
+ /// fmuladd (retried with a multiplicand), the first operand otherwise.
+ unsigned CopyableOpIdx = 0;
public:
Instruction *getMainOp() const {
@@ -721,7 +724,10 @@ class InstructionsState {
InstructionsState() = delete;
InstructionsState(Instruction *MainOp, Instruction *AltOp,
bool HasCopyables = false)
- : MainOp(MainOp), AltOp(AltOp), HasCopyables(HasCopyables) {}
+ : MainOp(MainOp), AltOp(AltOp), HasCopyables(HasCopyables),
+ CopyableOpIdx(MainOp && RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)
+ ? 2
+ : 0) {}
static InstructionsState invalid() { return {nullptr, nullptr}; }
/// Checks if the value is a copyable element.
@@ -828,6 +834,18 @@ class InstructionsState {
assert(valid() && "InstructionsState is invalid.");
return HasCopyables;
}
+
+ /// Returns the index of the operand the copyable value is modeled in.
+ unsigned getCopyableOpIdx() const {
+ assert(valid() && "InstructionsState is invalid.");
+ return CopyableOpIdx;
+ }
+
+ /// Sets the index of the operand the copyable value is modeled in.
+ void setCopyableOpIdx(unsigned Idx) {
+ assert((Idx == 0 || Idx == 2) && "Unexpected copyable operand index.");
+ CopyableOpIdx = Idx;
+ }
};
std::pair<Instruction *, SmallVector<Value *>>
@@ -11460,9 +11478,12 @@ class InstructionsCompatibilityAnalysis {
return {V, V};
if (!S.isCopyableElement(V))
return convertTo(cast<Instruction>(V), S).second;
- // fmuladd(0.0, -0.0, V) == V.
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)) {
Type *Ty = MainOp->getType();
+ // fmuladd(V, 1.0, -0.0) == V.
+ if (S.getCopyableOpIdx() == 0)
+ return {V, ConstantFP::get(Ty, 1.0), ConstantFP::getNegativeZero(Ty)};
+ // fmuladd(0.0, -0.0, V) == V.
return {ConstantFP::getZero(Ty), ConstantFP::getNegativeZero(Ty), V};
}
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
@@ -11985,7 +12006,7 @@ class InstructionsCompatibilityAnalysis {
return OrigS;
return S;
}
- // fmuladd is the only 3-operand copyable; the value sits in the addend.
+ // fmuladd is the only 3-operand copyable.
assert((Operands.size() == 2 ||
(Operands.size() == 3 &&
RecurrenceDescriptor::isFMulAddIntrinsic(MainOp))) &&
@@ -12038,8 +12059,9 @@ class InstructionsCompatibilityAnalysis {
}
}
}
- // 2. Check, if operands can be vectorized. Skip for fmuladd; its addend
- // is checked below and may legitimately hold many instructions.
+ // 2. Check, if operands can be vectorized. Skip for fmuladd; the
+ // copyable operand is checked below and may legitimately hold many
+ // instructions.
if (Operands.size() == 2 &&
count_if(Operands.back(), IsaPred<Instruction>) > 1)
return OrigS;
@@ -12076,10 +12098,16 @@ class InstructionsCompatibilityAnalysis {
count_if(Ops, [&](Value *V) { return OpS.isCopyableElement(V); });
return CopyableNum <= VL.size() / 2;
};
- // Check the addend for fmuladd, first operand otherwise.
- if (!CheckOperand(Operands.size() == 2 ? Operands.front()
- : Operands.back()))
- return OrigS;
+ // Check the operand holding the copyable values.
+ if (!CheckOperand(Operands[S.getCopyableOpIdx()])) {
+ if (Operands.size() == 2)
+ return OrigS;
+ // Retry with the copyable modeled as the first multiplicand.
+ S.setCopyableOpIdx(0);
+ Operands = buildOperands(S, VL);
+ if (!CheckOperand(Operands[S.getCopyableOpIdx()]))
+ return OrigS;
+ }
return S;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-mul-part.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-mul-part.ll
index 5fda54820f732..90194599b530c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-mul-part.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-mul-part.ll
@@ -8,18 +8,17 @@ define void @test_mul_copyable(ptr %dst, ptr %srcB, float %p, float %q, float %r
; ENABLED-LABEL: define void @test_mul_copyable(
; ENABLED-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[V:%.*]], float [[W:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]]) {
; ENABLED-NEXT: [[ENTRY:.*:]]
-; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
-; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
-; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
-; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
-; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
-; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
-; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[V]], i64 1
-; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
-; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[W]], i64 1
-; ENABLED-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP6]], [[TMP8]]
-; ENABLED-NEXT: [[TMP10:%.*]] = fdiv <2 x float> [[TMP6]], [[TMP8]]
-; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
+; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[P]], i64 0
+; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[R]], i64 1
+; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[T]], i64 2
+; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[V]], i64 3
+; ENABLED-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[Q]], i64 0
+; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[S]], i64 1
+; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[U]], i64 2
+; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[W]], i64 3
+; ENABLED-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP3]], [[TMP7]]
+; ENABLED-NEXT: [[TMP9:%.*]] = fdiv <4 x float> [[TMP3]], [[TMP7]]
+; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[G]], i64 1
; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
@@ -28,10 +27,12 @@ define void @test_mul_copyable(ptr %dst, ptr %srcB, float %p, float %q, float %r
; ENABLED-NEXT: [[TMP17:%.*]] = fsub <2 x float> [[TMP13]], [[TMP15]]
; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> [[TMP17]], <2 x i32> <i32 0, i32 3>
; ENABLED-NEXT: [[TMP19:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; ENABLED-NEXT: [[TMP20:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP19]], <2 x float> [[TMP18]])
-; ENABLED-NEXT: store <2 x float> [[TMP20]], ptr [[DST]], align 4
-; ENABLED-NEXT: [[D2:%.*]] = getelementptr float, ptr [[DST]], i32 2
-; ENABLED-NEXT: store <2 x float> [[TMP11]], ptr [[D2]], align 4
+; ENABLED-NEXT: [[TMP24:%.*]] = shufflevector <2 x float> [[TMP19]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP24]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP21]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP10]], <4 x float> [[TMP20]], <4 x float> [[TMP22]])
+; ENABLED-NEXT: store <4 x float> [[TMP23]], ptr [[DST]], align 4
; ENABLED-NEXT: ret void
;
; DISABLED-LABEL: define void @test_mul_copyable(
@@ -89,11 +90,8 @@ define void @test_mul_copyable_args(ptr %dst, ptr %srcB, float %p, float %q, flo
; ENABLED-LABEL: define void @test_mul_copyable_args(
; ENABLED-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[X:%.*]], float [[Y:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]]) {
; ENABLED-NEXT: [[ENTRY:.*:]]
-; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
-; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
-; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
-; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
-; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
+; ENABLED-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
+; ENABLED-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]]
; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[G]], i64 1
; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
@@ -102,12 +100,16 @@ define void @test_mul_copyable_args(ptr %dst, ptr %srcB, float %p, float %q, flo
; ENABLED-NEXT: [[TMP10:%.*]] = fsub <2 x float> [[TMP6]], [[TMP8]]
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
; ENABLED-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; ENABLED-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP12]], <2 x float> [[TMP11]])
-; ENABLED-NEXT: store <2 x float> [[TMP13]], ptr [[DST]], align 4
-; ENABLED-NEXT: [[D2:%.*]] = getelementptr float, ptr [[DST]], i32 2
-; ENABLED-NEXT: store float [[X]], ptr [[D2]], align 4
-; ENABLED-NEXT: [[D3:%.*]] = getelementptr float, ptr [[DST]], i32 3
-; ENABLED-NEXT: store float [[Y]], ptr [[D3]], align 4
+; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> poison, float [[X]], i64 2
+; ENABLED-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP17]], float [[Y]], i64 3
+; ENABLED-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[A0]], i64 0
+; ENABLED-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[A1]], i64 1
+; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP21]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP14]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP16:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP20]], <4 x float> [[TMP13]], <4 x float> [[TMP15]])
+; ENABLED-NEXT: store <4 x float> [[TMP16]], ptr [[DST]], align 4
; ENABLED-NEXT: ret void
;
; DISABLED-LABEL: define void @test_mul_copyable_args(
More information about the llvm-commits
mailing list