[llvm] adba9da - [SLP]Support copyable fadds in fmuladd, modeled as fmuladd(1.0, a, b)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 06:43:46 PDT 2026
Author: Alexey Bataev
Date: 2026-08-04T09:43:40-04:00
New Revision: adba9da44faf9c8a4b8695623023c265ffe59301
URL: https://github.com/llvm/llvm-project/commit/adba9da44faf9c8a4b8695623023c265ffe59301
DIFF: https://github.com/llvm/llvm-project/commit/adba9da44faf9c8a4b8695623023c265ffe59301.diff
LOG: [SLP]Support copyable fadds in fmuladd, modeled as fmuladd(1.0, a, b)
A copyable single-use fadd a, b is modeled as fmuladd(1.0, a, b), which
equals fadd a, b (the multiply by 1.0 is exact and preserves signed
zeros), so the add dies instead of being computed and gathered. The
addend/multiplicand assignment per lane matches the majority operand
kinds of the non-copyable lanes. Same all-or-nothing and tie-break
rules as for absorbed fmuls.
Reviewers: hiraditya, bababuck, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/213786
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fadd.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 97c14e87ccdcc..800291941c0f2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11233,8 +11233,8 @@ class InstructionsCompatibilityAnalysis {
unsigned MainOpcode = 0;
Instruction *MainOp = nullptr;
/// Whether every copyable in the current value list is an absorbable
- /// single-use fmul. Computed once per buildInstructionsState call.
- bool AbsorbCopyableFMuls = false;
+ /// single-use fmul/fadd. Computed once per buildInstructionsState call.
+ bool AbsorbCopyableFMulOrFAdds = false;
/// Checks if the opcode is supported as the main opcode for copyable
/// elements.
@@ -11342,14 +11342,18 @@ class InstructionsCompatibilityAnalysis {
MainBOOp1->getParent() == I->getParent())
continue;
}
- // Keep fmuladd over fmul on a tie only when every copyable is an
- // absorbed fmul.
+ // Keep fmuladd over fmul/fadd on a tie only when every copyable is
+ // an absorbed fmul/fadd.
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) &&
- I->getOpcode() == Instruction::FMul && AbsorbCopyableFMuls)
+ (I->getOpcode() == Instruction::FMul ||
+ I->getOpcode() == Instruction::FAdd) &&
+ AbsorbCopyableFMulOrFAdds)
continue;
- // Same check when fmuladd replaces fmul on a tie.
- if (MainOp->getOpcode() == Instruction::FMul &&
- RecurrenceDescriptor::isFMulAddIntrinsic(I) && !AbsorbCopyableFMuls)
+ // Same check when fmuladd replaces fmul/fadd on a tie.
+ if ((MainOp->getOpcode() == Instruction::FMul ||
+ MainOp->getOpcode() == Instruction::FAdd) &&
+ RecurrenceDescriptor::isFMulAddIntrinsic(I) &&
+ !AbsorbCopyableFMulOrFAdds)
continue;
}
UsedOutside = PUsedOutside;
@@ -11386,24 +11390,6 @@ class InstructionsCompatibilityAnalysis {
!MainOp->isCommutative());
}
- /// Checks if every copyable in \p VL is an absorbable fmul: the multiplies
- /// die instead of being computed and gathered. Multiplicand order is
- /// normalized when the operands are built.
- static bool hasOnlyAbsorbableCopyableFMuls(ArrayRef<Value *> VL) {
- bool HasFMul = false;
- for (Value *V : VL) {
- if (isa<PoisonValue>(V))
- continue;
- auto *I = dyn_cast<Instruction>(V);
- if (I && RecurrenceDescriptor::isFMulAddIntrinsic(I))
- continue;
- if (!isAbsorbableFMul(VL, V))
- return false;
- HasFMul = true;
- }
- return HasFMul;
- }
-
/// Returns the value and operands for the \p V, considering if it is original
/// instruction and its actual operands should be returned, or it is a
/// copyable element and its should be represented as idempotent instruction.
@@ -11414,11 +11400,15 @@ class InstructionsCompatibilityAnalysis {
return convertTo(cast<Instruction>(V), S).second;
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)) {
Type *Ty = MainOp->getType();
- // fmuladd(a, b, -0.0) == fmul a, b.
- if (S.hasAbsorbedCopyableFMul() && isAbsorbableCopyableFMul(S, V)) {
+ if (S.hasAbsorbedCopyableFMulOrFAdd() &&
+ isAbsorbableCopyableFMulOrFAdd(S, V)) {
auto *I = cast<Instruction>(V);
- return {I->getOperand(0), I->getOperand(1),
- ConstantFP::getNegativeZero(Ty)};
+ // fmuladd(a, b, -0.0) == fmul a, b.
+ if (I->getOpcode() == Instruction::FMul)
+ return {I->getOperand(0), I->getOperand(1),
+ ConstantFP::getNegativeZero(Ty)};
+ // fmuladd(1.0, a, b) == fadd a, b.
+ return {ConstantFP::get(Ty, 1.0), I->getOperand(0), I->getOperand(1)};
}
// fmuladd(V, 1.0, -0.0) == V.
if (S.getCopyableOpIdx() == 0)
@@ -11870,7 +11860,7 @@ class InstructionsCompatibilityAnalysis {
}
if (!VectorizeCopyableElements)
return S;
- AbsorbCopyableFMuls = hasOnlyAbsorbableCopyableFMuls(VL);
+ AbsorbCopyableFMulOrFAdds = hasOnlyAbsorbableCopyableFMulOrFAdds(VL);
findAndSetMainInstruction(VL, R);
if (!MainOp)
return S;
@@ -11883,11 +11873,12 @@ class InstructionsCompatibilityAnalysis {
// Check if it is profitable to vectorize the instruction.
unsigned CopyableNum =
count_if(VL, [&](Value *V) { return S.isCopyableElement(V); });
- // Absorb copyable single-use fmuls as fmuladd(a, b, -0.0) when every
- // copyable is such an fmul: the multiplies die instead of being computed
- // and gathered.
- if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) && AbsorbCopyableFMuls)
- S.setAbsorbCopyableFMul(true);
+ // Absorb copyable single-use fmuls/fadds as fmuladd(a, b, -0.0) or
+ // fmuladd(1.0, a, b) when every copyable is such a binop: the binops die
+ // instead of being computed and gathered.
+ if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) &&
+ AbsorbCopyableFMulOrFAdds)
+ S.setAbsorbCopyableFMulOrFAdd(true);
SmallVector<BoUpSLP::ValueList> Operands = buildOperands(S, VL);
auto BuildCandidates =
[](SmallVectorImpl<std::pair<Value *, Value *>> &Candidates, Value *V1,
@@ -12084,27 +12075,29 @@ class InstructionsCompatibilityAnalysis {
// Operand-order normalization below swaps OpIdx 0 and OpIdx 1
// of non-copyable lanes. That is only safe when the main op is
// commutative (e.g. 0 - X is not X - 0, so `sub` must be
- // excluded). With absorbed fmul copyables the fmuladd
+ // excluded). With absorbed fmul/fadd copyables the fmuladd
// multiplicands are commutative per lane and get normalized too;
- // the addend column is never touched.
- if (IsCommutative || S.hasAbsorbedCopyableFMul()) {
+ // the 0/1 swaps never touch the addend column.
+ if (IsCommutative || S.hasAbsorbedCopyableFMulOrFAdd()) {
// IsCommutative can hold for MainOp (e.g. a Sub/FSub feeding only
// fabs/icmp-eq-0) without every lane sharing that property, so
- // re-check the specific lane before swapping it. Absorbed fmul
+ // re-check the specific lane before swapping it. Absorbed fmul/fadd
// lanes are always commutative.
auto CanSwap = [&](Value *V) {
- if (S.hasAbsorbedCopyableFMul() && isAbsorbableCopyableFMul(S, V))
+ if (S.hasAbsorbedCopyableFMulOrFAdd() &&
+ isAbsorbableCopyableFMulOrFAdd(S, V))
return true;
return isCommutative(S.getMatchingMainOpOrAltOp(cast<Instruction>(V)),
V);
};
- // Absorbed fmul copyables do not vote for the majority operand
+ // Absorbed fmul/fadd copyables do not vote for the majority operand
// pattern (their multiplicand order is arbitrary) but take part
// in the swaps.
auto SwappableLane = [&](Value *V) {
return !isa<PoisonValue>(V) &&
- (!S.isCopyableElement(V) || (S.hasAbsorbedCopyableFMul() &&
- isAbsorbableCopyableFMul(S, V)));
+ (!S.isCopyableElement(V) ||
+ (S.hasAbsorbedCopyableFMulOrFAdd() &&
+ isAbsorbableCopyableFMulOrFAdd(S, V)));
};
// Count (ID0, ID1) pair frequencies for operand normalization.
// Pairs and their inverses are tracked under a canonical key
@@ -12115,12 +12108,15 @@ class InstructionsCompatibilityAnalysis {
unsigned RevCount = 0;
};
SmallMapVector<std::pair<unsigned, unsigned>, PairInfo, 8> PairCounts;
+ SmallMapVector<unsigned, unsigned, 4> AddendIDCounts;
unsigned MajID0 = 0, MajID1 = 0;
for (auto [Idx, V] : enumerate(VL)) {
if (S.isCopyableElement(V) || isa<PoisonValue>(V))
continue;
unsigned ID0 = Operands[0][Idx]->getValueID();
unsigned ID1 = Operands[1][Idx]->getValueID();
+ if (S.hasAbsorbedCopyableFMulOrFAdd())
+ ++AddendIDCounts[Operands[2][Idx]->getValueID()];
if (ID0 == ID1)
continue;
unsigned MinID = std::min(ID0, ID1);
@@ -12151,6 +12147,31 @@ class InstructionsCompatibilityAnalysis {
}
}
}
+ // Absorbed fadd copyables are fmuladd(1.0, a, b): pick the
+ // addend/multiplicand assignment matching the majority operand
+ // kinds of the non-copyable lanes in the multiplicand (1) and
+ // addend (2) columns.
+ if (S.hasAbsorbedCopyableFMulOrFAdd()) {
+ unsigned MajID2 = 0, Best2 = 0;
+ for (const auto &P : AddendIDCounts) {
+ if (P.second > Best2) {
+ Best2 = P.second;
+ MajID2 = P.first;
+ }
+ }
+ for (auto [Idx, V] : enumerate(VL)) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || I->getOpcode() != Instruction::FAdd ||
+ !isAbsorbableCopyableFMulOrFAdd(S, I))
+ continue;
+ unsigned ID1 = Operands[1][Idx]->getValueID();
+ unsigned ID2 = Operands[2][Idx]->getValueID();
+ unsigned Cur = (ID1 == MajID1) + (ID2 == MajID2);
+ unsigned Swapped = (ID2 == MajID1) + (ID1 == MajID2);
+ if (Swapped > Cur)
+ std::swap(Operands[1][Idx], Operands[2][Idx]);
+ }
+ }
// Normalize swappable lanes in two steps:
// 1) Swap lanes whose operand types are the exact inverse of
// the majority pattern, making the non-copyable lanes
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index e8388f7e66a5e..4388c040ce8fe 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -399,19 +399,39 @@ bool InstructionsState::isCopyableElement(Value *V) const {
!Converter.hasCandidateOpcode(getOpcode());
}
-bool isAbsorbableFMul(ArrayRef<Value *> VL, Value *V) {
+bool isAbsorbableFMulOrFAdd(ArrayRef<Value *> VL, Value *V) {
auto *I = dyn_cast<Instruction>(V);
- return I && I->getOpcode() == Instruction::FMul && I->hasOneUse() &&
- none_of(I->operands(),
- [&](Value *Op) { return is_contained(VL, Op); });
+ return I &&
+ (I->getOpcode() == Instruction::FMul ||
+ I->getOpcode() == Instruction::FAdd) &&
+ I->hasOneUse() && none_of(I->operands(), [&](Value *Op) {
+ return is_contained(VL, Op);
+ });
}
-bool isAbsorbableCopyableFMul(const InstructionsState &S, Value *V) {
+bool isAbsorbableCopyableFMulOrFAdd(const InstructionsState &S, Value *V) {
auto *I = dyn_cast<Instruction>(V);
- return I && S.isCopyableElement(I) && I->getOpcode() == Instruction::FMul &&
+ return I && S.isCopyableElement(I) &&
+ (I->getOpcode() == Instruction::FMul ||
+ I->getOpcode() == Instruction::FAdd) &&
I->hasOneUse();
}
+bool hasOnlyAbsorbableCopyableFMulOrFAdds(ArrayRef<Value *> VL) {
+ bool HasFMulOrFAdd = false;
+ for (Value *V : VL) {
+ if (isa<PoisonValue>(V))
+ continue;
+ auto *I = dyn_cast<Instruction>(V);
+ if (I && RecurrenceDescriptor::isFMulAddIntrinsic(I))
+ continue;
+ if (!isAbsorbableFMulOrFAdd(VL, V))
+ return false;
+ HasFMulOrFAdd = true;
+ }
+ return HasFMulOrFAdd;
+}
+
bool InstructionsState::isExpandedBinOp(Value *V) const {
assert(valid() && "InstructionsState is invalid.");
if (isCopyableElement(V))
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index 06d85d43494e4..bc25b9f85e4e4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -158,9 +158,10 @@ class InstructionsState {
/// Index of the operand modeling the copyable values: the addend for
/// fmuladd (retried with a multiplicand), the first operand otherwise.
unsigned CopyableOpIdx = 0;
- /// Whether copyable single-use fmuls are modeled as fmuladd(a, b, -0.0),
- /// absorbing the multiply instead of computing and gathering its result.
- bool AbsorbCopyableFMul = false;
+ /// Whether copyable single-use fmuls/fadds are modeled as
+ /// fmuladd(a, b, -0.0)/fmuladd(1.0, a, b), absorbing the binop instead of
+ /// computing and gathering its result.
+ bool AbsorbCopyableFMulOrFAdd = false;
public:
Instruction *getMainOp() const {
@@ -263,22 +264,30 @@ class InstructionsState {
CopyableOpIdx = Idx;
}
- /// Checks if copyable fmuls are absorbed as fmuladd(a, b, -0.0).
- bool hasAbsorbedCopyableFMul() const {
+ /// Checks if copyable fmuls/fadds are absorbed as fmuladd(a, b, -0.0) or
+ /// fmuladd(1.0, a, b).
+ bool hasAbsorbedCopyableFMulOrFAdd() const {
assert(valid() && "InstructionsState is invalid.");
- return AbsorbCopyableFMul;
+ return AbsorbCopyableFMulOrFAdd;
}
- /// Sets the absorbed-fmul modeling for copyable fmuls.
- void setAbsorbCopyableFMul(bool Absorb) { AbsorbCopyableFMul = Absorb; }
+ /// Sets the absorbed-fmul/fadd modeling for copyable fmuls/fadds.
+ void setAbsorbCopyableFMulOrFAdd(bool Absorb) {
+ AbsorbCopyableFMulOrFAdd = Absorb;
+ }
};
-/// Checks if \p V is a single-use fmul with operands outside \p VL.
-bool isAbsorbableFMul(ArrayRef<Value *> VL, Value *V);
+/// Checks if \p V is a single-use fmul/fadd with operands outside \p VL.
+bool isAbsorbableFMulOrFAdd(ArrayRef<Value *> VL, Value *V);
+
+/// Checks if \p V is a copyable single-use fmul/fadd, absorbable as
+/// fmuladd(a, b, -0.0) or fmuladd(1.0, a, b).
+bool isAbsorbableCopyableFMulOrFAdd(const InstructionsState &S, Value *V);
-/// Checks if \p V is a copyable single-use fmul, absorbable as
-/// fmuladd(a, b, -0.0).
-bool isAbsorbableCopyableFMul(const InstructionsState &S, Value *V);
+/// Checks if every copyable in \p VL is an absorbable fmul/fadd: the binops
+/// die instead of being computed and gathered. Operand order is normalized
+/// when the operands are built.
+bool hasOnlyAbsorbableCopyableFMulOrFAdds(ArrayRef<Value *> VL);
} // namespace llvm::slpvectorizer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fadd.ll
index 557f1b923c75f..f19b3cd2753e0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fadd.ll
@@ -9,28 +9,25 @@ define <4 x float> @buildvec_fadd_absorb(float %p, float %q, float %r, float %s,
; ENABLED-LABEL: define <4 x float> @buildvec_fadd_absorb(
; ENABLED-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
; ENABLED-NEXT: [[ENTRY:.*:]]
-; ENABLED-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
-; ENABLED-NEXT: [[C1:%.*]] = fmul float [[R]], [[S]]
-; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
-; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[G]], i64 1
-; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
-; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[H]], i64 1
-; ENABLED-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP1]], [[TMP3]]
-; ENABLED-NEXT: [[TMP7:%.*]] = fsub <2 x float> [[TMP1]], [[TMP3]]
-; ENABLED-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
+; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
+; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
+; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
+; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
+; ENABLED-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]]
+; ENABLED-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]]
; ENABLED-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
+; ENABLED-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP6]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <4 x float> poison, float [[T]], i64 2
; ENABLED-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[N]], i64 3
-; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP9]], float [[A0]], i64 0
+; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP10]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <4 x float> poison, float [[U]], i64 2
+; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP12]], float [[M]], i64 3
+; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[C0]], i64 0
; ENABLED-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[C1]], i64 1
-; ENABLED-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; ENABLED-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP19]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; ENABLED-NEXT: [[TMP21:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP15]], <4 x float> [[TMP13]], <4 x float> [[TMP20]])
-; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, float [[U]], i64 2
-; ENABLED-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP17]], float [[M]], i64 3
-; ENABLED-NEXT: [[TMP16:%.*]] = fadd <4 x float> [[TMP21]], [[TMP18]]
+; ENABLED-NEXT: [[TMP16:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP7]], <4 x float> [[TMP11]], <4 x float> [[TMP15]])
; ENABLED-NEXT: ret <4 x float> [[TMP16]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fadd_absorb(
@@ -109,22 +106,24 @@ define <4 x float> @buildvec_fadd_absorb_rev(float %p, float %q, float %r, float
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
+; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
+; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[G]], i64 1
+; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
+; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[H]], i64 1
+; ENABLED-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[TMP6]], [[TMP8]]
+; ENABLED-NEXT: [[TMP10:%.*]] = fsub <2 x float> [[TMP6]], [[TMP8]]
+; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
+; ENABLED-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
+; ENABLED-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <4 x float> <float 1.000000e+00, float 1.000000e+00, float poison, float poison>, <4 x float> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP15:%.*]] = insertelement <4 x float> poison, float [[T]], i64 0
; ENABLED-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[N]], i64 1
-; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP16]], float [[E]], i64 2
-; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[G]], i64 3
+; ENABLED-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <4 x float> [[TMP16]], <4 x float> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP19:%.*]] = insertelement <4 x float> poison, float [[U]], i64 0
; ENABLED-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[M]], i64 1
-; ENABLED-NEXT: [[TMP24:%.*]] = insertelement <4 x float> [[TMP20]], float [[F]], i64 2
-; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP24]], float [[H]], i64 3
-; ENABLED-NEXT: [[TMP13:%.*]] = fadd <4 x float> [[TMP8]], [[TMP12]]
-; ENABLED-NEXT: [[TMP25:%.*]] = fsub <4 x float> [[TMP8]], [[TMP12]]
-; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP25]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
-; ENABLED-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; ENABLED-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <4 x float> <float 0.000000e+00, float 0.000000e+00, float poison, float poison>, <4 x float> [[TMP17]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, <4 x float> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP14]], <4 x float> [[TMP18]], <4 x float> [[TMP22]])
; ENABLED-NEXT: ret <4 x float> [[TMP23]]
;
@@ -207,17 +206,16 @@ define <4 x float> @buildvec_fadd_absorb_swap_ops(float %p, float %q, float %r,
; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
; ENABLED-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
; ENABLED-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCC]], align 4
-; ENABLED-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP10]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, float [[U]], i64 0
+; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <4 x float> poison, float [[U]], i64 0
; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP12]], float [[M]], i64 1
-; ENABLED-NEXT: [[TMP17:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP9]], <4 x float> [[TMP11]], <4 x float> [[TMP13]])
-; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, float [[T]], i64 2
+; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[T]], i64 2
; ENABLED-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[N]], i64 3
-; ENABLED-NEXT: [[TMP16:%.*]] = fadd <4 x float> [[TMP17]], [[TMP15]]
+; ENABLED-NEXT: [[TMP16:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP9]], <4 x float> [[TMP11]], <4 x float> [[TMP15]])
; ENABLED-NEXT: ret <4 x float> [[TMP16]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fadd_absorb_swap_ops(
@@ -379,33 +377,24 @@ define <4 x float> @buildvec_mixed_fmul_fadd_absorb(float %p, float %q, float %r
; ENABLED-LABEL: define <4 x float> @buildvec_mixed_fmul_fadd_absorb(
; ENABLED-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
; ENABLED-NEXT: [[ENTRY:.*:]]
-; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
-; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP5]], float [[R]], i64 1
-; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
-; ENABLED-NEXT: [[TMP19:%.*]] = insertelement <2 x float> [[TMP17]], float [[S]], i64 1
-; ENABLED-NEXT: [[TMP21:%.*]] = fmul <2 x float> [[TMP13]], [[TMP19]]
+; ENABLED-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
+; ENABLED-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]]
; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
-; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP0]], float [[G]], i64 1
-; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
-; ENABLED-NEXT: [[TMP22:%.*]] = insertelement <2 x float> [[TMP7]], float [[H]], i64 1
-; ENABLED-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[TMP6]], [[TMP22]]
-; ENABLED-NEXT: [[TMP23:%.*]] = fsub <2 x float> [[TMP6]], [[TMP22]]
-; ENABLED-NEXT: [[TMP26:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP23]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
-; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP12]], float [[N]], i64 1
-; ENABLED-NEXT: [[TMP27:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
-; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP27]], float [[M]], i64 1
-; ENABLED-NEXT: [[TMP16:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]
+; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[M]], i64 1
+; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> <float poison, float -0.000000e+00>, float [[F]], i64 0
; ENABLED-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[TMP1]], [[TMP2]]
-; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; ENABLED-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]]
; ENABLED-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <2 x float> [[TMP21]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP8:%.*]] = shufflevector <4 x float> <float poison, float poison, float 0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP20]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[T]], i64 2
+; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[N]], i64 3
+; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[A0]], i64 0
+; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[A1]], i64 1
+; ENABLED-NEXT: [[TMP9:%.*]] = insertelement <4 x float> <float poison, float poison, float poison, float 1.000000e+00>, float [[U]], i64 2
; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float 0.000000e+00>, <4 x float> [[TMP10]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; ENABLED-NEXT: [[TMP24:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <4 x float> [[TMP24]], <4 x float> [[TMP25]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP10]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 1>
+; ENABLED-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float poison>, <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 poison, i32 2, i32 7>
+; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[C1]], i64 1
; ENABLED-NEXT: [[TMP15:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP8]], <4 x float> [[TMP11]], <4 x float> [[TMP14]])
; ENABLED-NEXT: ret <4 x float> [[TMP15]]
;
@@ -441,22 +430,22 @@ define <4 x float> @buildvec_mixed_fmul_fadd_absorb(float %p, float %q, float %r
; COST-LABEL: define <4 x float> @buildvec_mixed_fmul_fadd_absorb(
; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
; COST-NEXT: [[ENTRY:.*:]]
-; COST-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
-; COST-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]]
-; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]]
-; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]]
+; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
+; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
+; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
+; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
+; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]]
; COST-NEXT: [[Z:%.*]] = fadd float [[N]], [[M]]
+; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]]
+; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]]
; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; COST-NEXT: [[TMP1:%.*]] = insertelement <4 x float> <float poison, float poison, float 0.000000e+00, float -0.000000e+00>, float [[A0]], i64 0
-; COST-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A1]], i64 1
-; COST-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; COST-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float 0.000000e+00>, <4 x float> [[TMP3]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; COST-NEXT: [[TMP9:%.*]] = insertelement <4 x float> poison, float [[C0]], i64 0
-; COST-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP9]], float [[C1]], i64 1
-; COST-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[X]], i64 2
-; COST-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[Z]], i64 3
-; COST-NEXT: [[V3:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP2]], <4 x float> [[TMP4]], <4 x float> [[TMP8]])
+; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0
+; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1
+; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]])
+; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; COST-NEXT: [[V2:%.*]] = insertelement <4 x float> [[TMP9]], float [[X]], i32 2
+; COST-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Z]], i32 3
; COST-NEXT: ret <4 x float> [[V3]]
;
entry:
More information about the llvm-commits
mailing list