[llvm] f338032 - [SLP]Support copyable fmuls in fmuladd, modeled as fmuladd(a, b, -0.0)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 11:48:03 PDT 2026
Author: Alexey Bataev
Date: 2026-08-03T14:47:58-04:00
New Revision: f338032ff6f5c4d082cde3d9e89c95afc3b6982a
URL: https://github.com/llvm/llvm-project/commit/f338032ff6f5c4d082cde3d9e89c95afc3b6982a
DIFF: https://github.com/llvm/llvm-project/commit/f338032ff6f5c4d082cde3d9e89c95afc3b6982a.diff
LOG: [SLP]Support copyable fmuls in fmuladd, modeled as fmuladd(a, b, -0.0)
A copyable lane holding a single-use fmul a, b is modeled as
fmuladd(a, b, -0.0), which equals fmul a, b (the add of -0.0 is exact
and preserves signed zeros), so the multiply dies instead of being
computed and gathered. Applied only when every copyable lane is such
an fmul; multi-use fmuls and mixed copyables keep the
addend/multiplicand modeling. On a tie between fmuladd and fmul main
ops, fmuladd is preferred only when the fmuls are absorbed profitably:
single-use, operands not part of the list and vectorizable as
multiplicand operands.
Reviewers: bababuck, hiraditya, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/213369
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4bf161d0f9c0d..1c9d10db6d241 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11212,6 +11212,9 @@ class InstructionsCompatibilityAnalysis {
const TargetLibraryInfo &TLI;
unsigned MainOpcode = 0;
Instruction *MainOp = nullptr;
+ /// Whether every copyable in the current value list is an absorbable
+ /// single-use fmul. Computed once per buildInstructionsState call.
+ bool AbsorbCopyableFMuls = false;
/// Checks if the opcode is supported as the main opcode for copyable
/// elements.
@@ -11319,6 +11322,15 @@ class InstructionsCompatibilityAnalysis {
MainBOOp1->getParent() == I->getParent())
continue;
}
+ // Keep fmuladd over fmul on a tie only when every copyable is an
+ // absorbed fmul.
+ if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) &&
+ I->getOpcode() == Instruction::FMul && AbsorbCopyableFMuls)
+ continue;
+ // Same check when fmuladd replaces fmul on a tie.
+ if (MainOp->getOpcode() == Instruction::FMul &&
+ RecurrenceDescriptor::isFMulAddIntrinsic(I) && !AbsorbCopyableFMuls)
+ continue;
}
UsedOutside = PUsedOutside;
for (Instruction *I : P.second) {
@@ -11354,6 +11366,24 @@ class InstructionsCompatibilityAnalysis {
!MainOp->isCommutative());
}
+ /// Checks if every copyable in \p VL is an absorbable fmul: the multiplies
+ /// die instead of being computed and gathered. Multiplicand order is
+ /// normalized when the operands are built.
+ static bool hasOnlyAbsorbableCopyableFMuls(ArrayRef<Value *> VL) {
+ bool HasFMul = false;
+ for (Value *V : VL) {
+ if (isa<PoisonValue>(V))
+ continue;
+ auto *I = dyn_cast<Instruction>(V);
+ if (I && RecurrenceDescriptor::isFMulAddIntrinsic(I))
+ continue;
+ if (!isAbsorbableFMul(VL, V))
+ return false;
+ HasFMul = true;
+ }
+ return HasFMul;
+ }
+
/// Returns the value and operands for the \p V, considering if it is original
/// instruction and its actual operands should be returned, or it is a
/// copyable element and its should be represented as idempotent instruction.
@@ -11364,6 +11394,12 @@ class InstructionsCompatibilityAnalysis {
return convertTo(cast<Instruction>(V), S).second;
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)) {
Type *Ty = MainOp->getType();
+ // fmuladd(a, b, -0.0) == fmul a, b.
+ if (S.hasAbsorbedCopyableFMul() && isAbsorbableCopyableFMul(S, V)) {
+ auto *I = cast<Instruction>(V);
+ return {I->getOperand(0), I->getOperand(1),
+ ConstantFP::getNegativeZero(Ty)};
+ }
// fmuladd(V, 1.0, -0.0) == V.
if (S.getCopyableOpIdx() == 0)
return {V, ConstantFP::get(Ty, 1.0), ConstantFP::getNegativeZero(Ty)};
@@ -11814,6 +11850,7 @@ class InstructionsCompatibilityAnalysis {
}
if (!VectorizeCopyableElements)
return S;
+ AbsorbCopyableFMuls = hasOnlyAbsorbableCopyableFMuls(VL);
findAndSetMainInstruction(VL, R);
if (!MainOp)
return S;
@@ -11824,6 +11861,13 @@ class InstructionsCompatibilityAnalysis {
if (!WithProfitabilityCheck)
return S;
// Check if it is profitable to vectorize the instruction.
+ unsigned CopyableNum =
+ count_if(VL, [&](Value *V) { return S.isCopyableElement(V); });
+ // Absorb copyable single-use fmuls as fmuladd(a, b, -0.0) when every
+ // copyable is such an fmul: the multiplies die instead of being computed
+ // and gathered.
+ if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) && AbsorbCopyableFMuls)
+ S.setAbsorbCopyableFMul(true);
SmallVector<BoUpSLP::ValueList> Operands = buildOperands(S, VL);
auto BuildCandidates =
[](SmallVectorImpl<std::pair<Value *, Value *>> &Candidates, Value *V1,
@@ -11895,8 +11939,6 @@ class InstructionsCompatibilityAnalysis {
(Operands.size() == 3 &&
RecurrenceDescriptor::isFMulAddIntrinsic(MainOp))) &&
"Unexpected number of operands!");
- unsigned CopyableNum =
- count_if(VL, [&](Value *V) { return S.isCopyableElement(V); });
if (CopyableNum < VL.size() / 2)
return S;
// Too many phi copyables - exit.
@@ -12022,15 +12064,28 @@ class InstructionsCompatibilityAnalysis {
// Operand-order normalization below swaps OpIdx 0 and OpIdx 1
// of non-copyable lanes. That is only safe when the main op is
// commutative (e.g. 0 - X is not X - 0, so `sub` must be
- // excluded).
- if (IsCommutative) {
+ // excluded). With absorbed fmul copyables the fmuladd
+ // multiplicands are commutative per lane and get normalized too;
+ // the addend column is never touched.
+ if (IsCommutative || S.hasAbsorbedCopyableFMul()) {
// IsCommutative can hold for MainOp (e.g. a Sub/FSub feeding only
// fabs/icmp-eq-0) without every lane sharing that property, so
- // re-check the specific lane before swapping it.
+ // re-check the specific lane before swapping it. Absorbed fmul
+ // lanes are always commutative.
auto CanSwap = [&](Value *V) {
+ if (S.hasAbsorbedCopyableFMul() && isAbsorbableCopyableFMul(S, V))
+ return true;
return isCommutative(S.getMatchingMainOpOrAltOp(cast<Instruction>(V)),
V);
};
+ // Absorbed fmul copyables do not vote for the majority operand
+ // pattern (their multiplicand order is arbitrary) but take part
+ // in the swaps.
+ auto SwappableLane = [&](Value *V) {
+ return !isa<PoisonValue>(V) &&
+ (!S.isCopyableElement(V) || (S.hasAbsorbedCopyableFMul() &&
+ isAbsorbableCopyableFMul(S, V)));
+ };
// Count (ID0, ID1) pair frequencies for operand normalization.
// Pairs and their inverses are tracked under a canonical key
// so that (Load, Add) and (Add, Load) contribute to the same
@@ -12076,16 +12131,16 @@ class InstructionsCompatibilityAnalysis {
}
}
}
- // Normalize non-copyable lanes in two steps:
+ // Normalize swappable lanes in two steps:
// 1) Swap lanes whose operand types are the exact inverse of
// the majority pattern, making the non-copyable lanes
// consistent.
- // 2) Independently, if a strict majority of non-copyable lanes
+ // 2) Independently, if a strict majority of swappable lanes
// have loads at OpIdx 1, swap those lanes to put loads at
// OpIdx 0 for better downstream vectorization.
unsigned LAt0 = 0, LAt1 = 0, TotalNC = 0;
for (auto [Idx, V] : enumerate(VL)) {
- if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ if (!SwappableLane(V))
continue;
// Step 1: swap exact-inverse lanes.
if (BestCount > 0) {
@@ -12102,7 +12157,7 @@ class InstructionsCompatibilityAnalysis {
// swap those lanes to put loads at OpIdx 0.
if (TotalNC > 1 && LAt1 > LAt0 && LAt1 * 2 > TotalNC) {
for (auto [Idx, V] : enumerate(VL)) {
- if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ if (!SwappableLane(V))
continue;
if (!isa<LoadInst>(Operands[0][Idx]) &&
isa<LoadInst>(Operands[1][Idx]) && CanSwap(V))
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 54cb526a10eab..e8388f7e66a5e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -399,6 +399,19 @@ bool InstructionsState::isCopyableElement(Value *V) const {
!Converter.hasCandidateOpcode(getOpcode());
}
+bool isAbsorbableFMul(ArrayRef<Value *> VL, Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && I->getOpcode() == Instruction::FMul && I->hasOneUse() &&
+ none_of(I->operands(),
+ [&](Value *Op) { return is_contained(VL, Op); });
+}
+
+bool isAbsorbableCopyableFMul(const InstructionsState &S, Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && S.isCopyableElement(I) && I->getOpcode() == Instruction::FMul &&
+ I->hasOneUse();
+}
+
bool InstructionsState::isExpandedBinOp(Value *V) const {
assert(valid() && "InstructionsState is invalid.");
if (isCopyableElement(V))
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index 9f2bd8b5d6aca..06d85d43494e4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -158,6 +158,9 @@ class InstructionsState {
/// Index of the operand modeling the copyable values: the addend for
/// fmuladd (retried with a multiplicand), the first operand otherwise.
unsigned CopyableOpIdx = 0;
+ /// Whether copyable single-use fmuls are modeled as fmuladd(a, b, -0.0),
+ /// absorbing the multiply instead of computing and gathering its result.
+ bool AbsorbCopyableFMul = false;
public:
Instruction *getMainOp() const {
@@ -259,8 +262,24 @@ class InstructionsState {
assert((Idx == 0 || Idx == 2) && "Unexpected copyable operand index.");
CopyableOpIdx = Idx;
}
+
+ /// Checks if copyable fmuls are absorbed as fmuladd(a, b, -0.0).
+ bool hasAbsorbedCopyableFMul() const {
+ assert(valid() && "InstructionsState is invalid.");
+ return AbsorbCopyableFMul;
+ }
+
+ /// Sets the absorbed-fmul modeling for copyable fmuls.
+ void setAbsorbCopyableFMul(bool Absorb) { AbsorbCopyableFMul = Absorb; }
};
+/// Checks if \p V is a single-use fmul with operands outside \p VL.
+bool isAbsorbableFMul(ArrayRef<Value *> VL, Value *V);
+
+/// Checks if \p V is a copyable single-use fmul, absorbable as
+/// fmuladd(a, b, -0.0).
+bool isAbsorbableCopyableFMul(const InstructionsState &S, Value *V);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOMPATIBILITYANALYSIS_H
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll
index 28ca0b93a06f7..355b1d42d159b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll
@@ -23,15 +23,14 @@ define <4 x float> @buildvec_fmul_absorb(float %p, float %q, float %r, float %s,
; ENABLED-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[N]], i64 3
; ENABLED-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[A0]], i64 0
; ENABLED-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP10]], float [[A1]], i64 1
+; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <4 x float> poison, float [[U]], i64 2
+; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP12]], float [[M]], i64 3
; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP14]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP14]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; ENABLED-NEXT: [[TMP16:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP17:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP16]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; ENABLED-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP15]], <4 x float> [[TMP17]])
-; ENABLED-NEXT: [[TMP20:%.*]] = insertelement <4 x float> <float 1.000000e+00, float 1.000000e+00, float poison, float poison>, float [[U]], i64 2
-; ENABLED-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[M]], i64 3
-; ENABLED-NEXT: [[TMP19:%.*]] = fmul <4 x float> [[TMP18]], [[TMP21]]
-; ENABLED-NEXT: ret <4 x float> [[TMP19]]
+; ENABLED-NEXT: ret <4 x float> [[TMP18]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fmul_absorb(
; DISABLED-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
@@ -104,13 +103,7 @@ define void @storechain_1fmul(ptr %dst, ptr %srcB, float %p, float %q, float %r,
; ENABLED-LABEL: define void @storechain_1fmul(
; ENABLED-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], float [[I:%.*]], float [[J:%.*]], float [[K:%.*]], float [[L:%.*]]) {
; ENABLED-NEXT: [[ENTRY:.*:]]
-; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
-; ENABLED-NEXT: [[TMP16:%.*]] = insertelement <2 x float> [[TMP7]], float [[E]], i64 1
-; ENABLED-NEXT: [[TMP24:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
-; ENABLED-NEXT: [[TMP25:%.*]] = insertelement <2 x float> [[TMP24]], float [[F]], i64 1
-; ENABLED-NEXT: [[TMP26:%.*]] = fmul <2 x float> [[TMP16]], [[TMP25]]
-; ENABLED-NEXT: [[TMP27:%.*]] = fadd <2 x float> [[TMP16]], [[TMP25]]
-; ENABLED-NEXT: [[TMP28:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> [[TMP27]], <2 x i32> <i32 0, i32 3>
+; ENABLED-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]]
; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[G]], i64 0
; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[K]], i64 1
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[H]], i64 0
@@ -120,7 +113,8 @@ define void @storechain_1fmul(ptr %dst, ptr %srcB, float %p, float %q, float %r,
; ENABLED-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x i32> <i32 0, i32 3>
; ENABLED-NEXT: [[B0:%.*]] = load float, ptr [[SRCB]], align 4
; ENABLED-NEXT: [[GEPB1:%.*]] = getelementptr float, ptr [[SRCB]], i32 1
-; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, float [[P]], i64 1
+; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[U]], i64 0
+; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[P]], i64 1
; ENABLED-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[R]], i64 2
; ENABLED-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[I]], i64 3
; ENABLED-NEXT: [[TMP11:%.*]] = insertelement <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, float [[Q]], i64 1
@@ -128,13 +122,14 @@ define void @storechain_1fmul(ptr %dst, ptr %srcB, float %p, float %q, float %r,
; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP12]], float [[J]], i64 3
; ENABLED-NEXT: [[TMP14:%.*]] = fmul <4 x float> [[TMP10]], [[TMP13]]
; ENABLED-NEXT: [[TMP15:%.*]] = load <2 x float>, ptr [[GEPB1]], align 4
-; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> <float -0.000000e+00, float poison, float poison, float poison>, float [[B0]], i64 1
+; ENABLED-NEXT: [[TMP16:%.*]] = insertelement <4 x float> poison, float [[T]], i64 0
+; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP16]], float [[B0]], i64 1
; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <2 x float> [[TMP15]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <2 x float> [[TMP28]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP20:%.*]] = insertelement <4 x float> <float -0.000000e+00, float poison, float poison, float poison>, float [[C0]], i64 1
; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP14]], <4 x float> [[TMP19]], <4 x float> [[TMP22]])
+; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP19]], <4 x float> [[TMP14]], <4 x float> [[TMP22]])
; ENABLED-NEXT: store <4 x float> [[TMP23]], ptr [[DST]], align 4
; ENABLED-NEXT: ret void
;
@@ -237,20 +232,18 @@ define <4 x float> @buildvec_fmul_absorb_rev(float %p, float %q, float %r, float
; ENABLED-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[TMP6]], [[TMP8]]
; ENABLED-NEXT: [[TMP10:%.*]] = fsub <2 x float> [[TMP6]], [[TMP8]]
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
-; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP17]], float [[N]], i64 1
-; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
-; ENABLED-NEXT: [[TMP18:%.*]] = insertelement <2 x float> [[TMP14]], float [[M]], i64 1
-; ENABLED-NEXT: [[TMP16:%.*]] = fmul <2 x float> [[TMP13]], [[TMP18]]
; ENABLED-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
+; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[T]], i64 0
+; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[N]], i64 1
; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP19:%.*]] = shufflevector <4 x float> <float 0.000000e+00, float 0.000000e+00, float poison, float poison>, <4 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, <4 x float> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP26:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP16:%.*]] = shufflevector <4 x float> [[TMP14]], <4 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> poison, float [[U]], i64 0
+; ENABLED-NEXT: [[TMP25:%.*]] = insertelement <4 x float> [[TMP17]], float [[M]], i64 1
+; ENABLED-NEXT: [[TMP26:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP24:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> [[TMP26]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP19]], <4 x float> [[TMP22]], <4 x float> [[TMP24]])
+; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, <4 x float> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP16]], <4 x float> [[TMP24]], <4 x float> [[TMP22]])
; ENABLED-NEXT: ret <4 x float> [[TMP23]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fmul_absorb_rev(
@@ -337,25 +330,24 @@ define <4 x float> @buildvec_fmul_no_absorb_multiuse_rev(float %p, float %q, flo
; ENABLED-NEXT: [[TMP9:%.*]] = fadd <2 x float> [[TMP6]], [[TMP8]]
; ENABLED-NEXT: [[TMP10:%.*]] = fsub <2 x float> [[TMP6]], [[TMP8]]
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
-; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP17]], float [[N]], i64 1
-; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
-; ENABLED-NEXT: [[TMP18:%.*]] = insertelement <2 x float> [[TMP14]], float [[M]], i64 1
-; ENABLED-NEXT: [[TMP16:%.*]] = fmul <2 x float> [[TMP13]], [[TMP18]]
; ENABLED-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
+; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[T]], i64 0
+; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[N]], i64 1
; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <4 x float> <float 0.000000e+00, float 0.000000e+00, float poison, float poison>, <4 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <4 x float> [[TMP14]], <4 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, <4 x float> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <4 x float> <float 1.000000e+00, float 1.000000e+00, float poison, float poison>, <4 x float> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP23:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP27:%.*]] = shufflevector <4 x float> [[TMP22]], <4 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; ENABLED-NEXT: [[TMP27:%.*]] = shufflevector <4 x float> <float -0.000000e+00, float -0.000000e+00, float poison, float poison>, <4 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; ENABLED-NEXT: [[TMP24:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP21]], <4 x float> [[TMP20]], <4 x float> [[TMP27]])
-; ENABLED-NEXT: [[TMP25:%.*]] = extractelement <2 x float> [[TMP16]], i64 0
+; ENABLED-NEXT: [[TMP22:%.*]] = insertelement <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, float [[U]], i64 0
+; ENABLED-NEXT: [[TMP28:%.*]] = insertelement <4 x float> [[TMP22]], float [[M]], i64 1
+; ENABLED-NEXT: [[TMP29:%.*]] = fmul <4 x float> [[TMP24]], [[TMP28]]
+; ENABLED-NEXT: [[TMP25:%.*]] = extractelement <4 x float> [[TMP29]], i64 0
; ENABLED-NEXT: store float [[TMP25]], ptr [[DST2]], align 4
-; ENABLED-NEXT: [[TMP26:%.*]] = extractelement <2 x float> [[TMP16]], i64 1
+; ENABLED-NEXT: [[TMP26:%.*]] = extractelement <4 x float> [[TMP29]], i64 1
; ENABLED-NEXT: store float [[TMP26]], ptr [[DST2]], align 4
-; ENABLED-NEXT: ret <4 x float> [[TMP24]]
+; ENABLED-NEXT: ret <4 x float> [[TMP29]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fmul_no_absorb_multiuse_rev(
; DISABLED-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]], ptr [[DST2:%.*]]) {
@@ -454,12 +446,11 @@ define <4 x float> @buildvec_fmul_absorb_const(float %p, float %q, float %r, flo
; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP12]], float [[A1]], i64 1
; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; ENABLED-NEXT: [[TMP16:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP15]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; ENABLED-NEXT: [[TMP16:%.*]] = shufflevector <4 x float> <float poison, float poison, float 2.000000e+00, float 4.000000e+00>, <4 x float> [[TMP15]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; ENABLED-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP17]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; ENABLED-NEXT: [[TMP19:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP14]], <4 x float> [[TMP16]], <4 x float> [[TMP18]])
-; ENABLED-NEXT: [[TMP20:%.*]] = fmul <4 x float> [[TMP19]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>
-; ENABLED-NEXT: ret <4 x float> [[TMP20]]
+; ENABLED-NEXT: ret <4 x float> [[TMP19]]
;
; DISABLED-LABEL: define <4 x float> @buildvec_fmul_absorb_const(
; DISABLED-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
@@ -489,21 +480,23 @@ define <4 x float> @buildvec_fmul_absorb_const(float %p, float %q, float %r, flo
; COST-LABEL: define <4 x float> @buildvec_fmul_absorb_const(
; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) {
; COST-NEXT: [[ENTRY:.*:]]
-; COST-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
-; COST-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]]
+; COST-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
+; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP8]], float [[R]], i64 1
+; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
+; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
+; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
+; COST-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
+; COST-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <2 x i32> zeroinitializer
+; COST-NEXT: [[TMP7:%.*]] = fmul <2 x float> [[TMP6]], <float 2.000000e+00, float 4.000000e+00>
; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]]
; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]]
; COST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
-; COST-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[U]], i64 2
-; COST-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A0]], i64 0
-; COST-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A1]], i64 1
-; COST-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
-; COST-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; COST-NEXT: [[TMP6:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; COST-NEXT: [[TMP7:%.*]] = insertelement <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, float [[C0]], i64 0
-; COST-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[C1]], i64 1
-; COST-NEXT: [[TMP9:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP4]], <4 x float> [[TMP6]], <4 x float> [[TMP8]])
-; COST-NEXT: [[TMP10:%.*]] = fmul <4 x float> [[TMP9]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>
+; COST-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0
+; COST-NEXT: [[TMP14:%.*]] = insertelement <2 x float> [[TMP9]], float [[C1]], i64 1
+; COST-NEXT: [[TMP11:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP0]], <2 x float> [[TMP14]])
+; COST-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; COST-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; COST-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> [[TMP12]], <4 x float> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; COST-NEXT: ret <4 x float> [[TMP10]]
;
entry:
More information about the llvm-commits
mailing list