[llvm] [SLP][X86]Vectorize alternate fadd/fsub fused with fmul (fmaddsub/fmsubadd) (PR #229268)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 7 05:26:19 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/229268
>From fc036d80a1640acaec5a039dbe9f33423242ab11 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 5 Oct 2026 16:49:08 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Analysis/TargetTransformInfo.h | 17 ++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 12 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 11 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 53 +++++--
llvm/lib/Target/X86/X86TargetTransformInfo.h | 12 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 53 ++++++-
.../SLPVectorizer/X86/fused-alt-fmul.ll | 144 ++++++++----------
7 files changed, 179 insertions(+), 123 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index e81809cca64f8..c245579196646 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -997,9 +997,13 @@ class TargetTransformInfo {
/// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0`
/// when \p Opcode0 is selected and `1` when Opcode1 is selected.
/// \p VecTy is the vector type of the instruction to be generated.
+ /// \p Scalars are the scalar instructions of the lanes, if known. The target
+ /// may use them to find the patterns, which depend on the operands, like
+ /// the alternating fused multiply-add.
LLVM_ABI bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
- const SmallBitVector &OpcodeMask) const;
+ const SmallBitVector &OpcodeMask,
+ ArrayRef<const Value *> Scalars = {}) const;
/// Return true if we should be enabling ordered reductions for the target.
LLVM_ABI bool enableOrderedReductions() const;
@@ -1598,10 +1602,13 @@ class TargetTransformInfo {
/// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0`
/// when \p Opcode0 is selected and `1` when Opcode1 is selected.
/// \p VecTy is the vector type of the instruction to be generated.
- LLVM_ABI InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
- unsigned Opcode1,
- const SmallBitVector &OpcodeMask,
- TTI::TargetCostKind CostKind) const;
+ /// \p Scalars are the scalar instructions of the lanes, if known. The target
+ /// may use them to find the patterns, which depend on the operands, like
+ /// the alternating fused multiply-add.
+ LLVM_ABI InstructionCost getAltInstrCost(
+ VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
+ const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind,
+ ArrayRef<const Value *> Scalars = {}) const;
/// \return The cost of a shuffle instruction of kind Kind with inputs of type
/// SrcTy, producing a vector of type DstTy. The exact mask may be passed as
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a19c122c16f20..5b3f9d3d91c26 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -417,7 +417,8 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
- const SmallBitVector &OpcodeMask) const {
+ const SmallBitVector &OpcodeMask,
+ ArrayRef<const Value *> Scalars) const {
return false;
}
@@ -775,10 +776,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
- unsigned Opcode1,
- const SmallBitVector &OpcodeMask,
- TTI::TargetCostKind CostKind) const {
+ virtual InstructionCost
+ getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
+ const SmallBitVector &OpcodeMask,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<const Value *> Scalars) const {
return InstructionCost::getInvalid();
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 1055bb59148b3..b588f15bb3a09 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -520,8 +520,8 @@ bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
bool TargetTransformInfo::isLegalAltInstr(
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
- const SmallBitVector &OpcodeMask) const {
- return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
+ const SmallBitVector &OpcodeMask, ArrayRef<const Value *> Scalars) const {
+ return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask, Scalars);
}
bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
@@ -1095,9 +1095,10 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
InstructionCost TargetTransformInfo::getAltInstrCost(
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
- const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
- InstructionCost Cost =
- TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
+ const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind,
+ ArrayRef<const Value *> Scalars) const {
+ InstructionCost Cost = TTIImpl->getAltInstrCost(
+ VecTy, Opcode0, Opcode1, OpcodeMask, CostKind, Scalars);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 393e5bfe45ffb..27a92d5eadcf0 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1939,8 +1939,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
InstructionCost
X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1, const SmallBitVector &OpcodeMask,
- TTI::TargetCostKind CostKind) const {
- if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask))
+ TTI::TargetCostKind CostKind,
+ ArrayRef<const Value *> Scalars) const {
+ if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask, Scalars))
return TTI::TCC_Basic;
return InstructionCost::getInvalid();
}
@@ -7407,7 +7408,8 @@ bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) const {
bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
- const SmallBitVector &OpcodeMask) const {
+ const SmallBitVector &OpcodeMask,
+ ArrayRef<const Value *> Scalars) const {
// ADDSUBPS 4xf32 SSE3
// VADDSUBPS 4xf32 AVX
// VADDSUBPS 8xf32 AVX2
@@ -7421,21 +7423,40 @@ bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
return false;
// Check the opcode pattern. We apply the mask on the opcode arguments and
// then check if it is what we expect.
- for (int Lane : seq<int>(0, NumElements)) {
- unsigned Opc = OpcodeMask.test(Lane) ? Opcode1 : Opcode0;
- // We expect FSub for even lanes and FAdd for odd lanes.
- if (Lane % 2 == 0 && Opc != Instruction::FSub)
- return false;
- if (Lane % 2 == 1 && Opc != Instruction::FAdd)
- return false;
- }
+ auto IsLaneOrder = [&](unsigned EvenOpc, unsigned OddOpc) {
+ return all_of(seq<unsigned>(0, NumElements), [&](unsigned Lane) {
+ return (OpcodeMask.test(Lane) ? Opcode1 : Opcode0) ==
+ (Lane % 2 == 0 ? EvenOpc : OddOpc);
+ });
+ };
+ // We expect FSub for even lanes and FAdd for odd lanes.
+ const bool IsAddSub = IsLaneOrder(Instruction::FSub, Instruction::FAdd);
// Now check that the pattern is supported by the target ISA.
Type *ElemTy = cast<VectorType>(VecTy)->getElementType();
- if (ElemTy->isFloatTy())
- return ST->hasSSE3() && NumElements % 4 == 0;
- if (ElemTy->isDoubleTy())
- return ST->hasSSE3() && NumElements % 2 == 0;
- return false;
+ if (IsAddSub && ST->hasSSE3() &&
+ ((ElemTy->isFloatTy() && NumElements % 4 == 0) ||
+ (ElemTy->isDoubleTy() && NumElements % 2 == 0)))
+ return true;
+ // The multiplication is fused into every lane: fmaddsub (the even lanes
+ // subtract and the odd lanes add the product) or fmsubadd (the other way
+ // round). The subtracted product is the first operand of the subtraction.
+ using namespace PatternMatch;
+ auto IsFusedWithFMul = [](const Value *V) {
+ const auto *I = dyn_cast<Instruction>(V);
+ if (!I || !I->hasAllowContract())
+ return false;
+ auto IsFMul = [&](const Value *Op) {
+ return match(Op,
+ m_OneUse(m_AllowContract(m_FMul(m_Value(), m_Value())))) &&
+ cast<Instruction>(Op)->getParent() == I->getParent();
+ };
+ return IsFMul(I->getOperand(0)) ||
+ (I->getOpcode() == Instruction::FAdd && IsFMul(I->getOperand(1)));
+ };
+ return ST->hasFMA() && !Scalars.empty() &&
+ (ElemTy->isFloatTy() || ElemTy->isDoubleTy()) &&
+ (IsAddSub || IsLaneOrder(Instruction::FAdd, Instruction::FSub)) &&
+ all_of(Scalars, IsFusedWithFMul);
}
bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 41fbcc77304fb..160a71e1affbb 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -72,10 +72,11 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
ArrayRef<const Value *> Args = {},
const Instruction *CtxI = nullptr) const override;
- InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
- unsigned Opcode1,
- const SmallBitVector &OpcodeMask,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost
+ getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
+ const SmallBitVector &OpcodeMask,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<const Value *> Scalars) const override;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
@@ -232,7 +233,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
bool isLegalMaskedCompressStore(Type *DataType,
Align Alignment) const override;
bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
- const SmallBitVector &OpcodeMask) const override;
+ const SmallBitVector &OpcodeMask,
+ ArrayRef<const Value *> Scalars) const override;
bool hasDivRemOp(Type *DataType, bool IsSigned) const override;
bool isExpensiveToSpeculativelyExecute(const Instruction *I) const override;
bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const override;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fb550f573508e..74b31042076a0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13523,6 +13523,32 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
uint64_t Total = 0;
+ // The alternate op, fused with the multiplication by the target, is counted
+ // with the multiplication node, the same way as the scalar fmas.
+ auto IsFusedAlt = [&](const TreeEntry &TE) {
+ if (!TE.hasState() || TE.State != TreeEntry::Vectorize ||
+ DeletedNodes.contains(&TE) || TransformedToGatherNodes.contains(&TE) ||
+ !TE.isAltShuffle() || TE.hasReassocScalars() ||
+ !TE.getMainOp()->getType()->isFloatingPointTy() ||
+ !TE.getOperations().isAddSubLikeOp())
+ return false;
+ Type *ScalarTy = TE.getMainOp()->getType();
+ auto *VecTy =
+ cast<VectorType>(getWidenedType(ScalarTy, TE.getVectorFactor()));
+ SmallBitVector OpcodeMask(getAltInstrMask(
+ TE.Scalars, ScalarTy, TE.getOpcode(), TE.getAltOpcode()));
+ return TTI->isLegalAltInstr(VecTy, TE.getOpcode(), TE.getAltOpcode(),
+ OpcodeMask, TE.Scalars) &&
+ canConvertToFMA(TE.Scalars, TE.getOperations(), *DT, *DL, *TTI, *TLI,
+ *this)
+ .isValid();
+ };
+ // The scalar count already credits the fused scalars as fmas, a tree with the
+ // fused alternate op is counted at the level of the machine instructions.
+ const bool HasFusedAlt =
+ any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &Ptr) {
+ return IsFusedAlt(*Ptr);
+ });
// Source vector -> max scale among the gather entries sharing it, so the
// combined shufflevector is still weighted like an in-loop entry below.
SmallDenseMap<Value *, uint64_t, 4> GatherExtractSourceVecs;
@@ -13563,10 +13589,11 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
VecScale = std::max(VecScale, Scale);
}
} else {
- for (Value *V : TE.Scalars) {
- if (!isConstant(V))
- ++Count;
- }
+ // A splat is a single broadcast.
+ if (HasFusedAlt && isSplat(TE.Scalars))
+ Count = !isConstant(TE.Scalars.front());
+ else
+ Count = TE.Scalars.size() - count_if(TE.Scalars, isConstant);
}
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
@@ -13587,6 +13614,8 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
+ if (IsFusedAlt(TE))
+ continue;
if (TE.State == TreeEntry::SplitVectorize)
Count += 2;
else if (TE.hasReassocScalars())
@@ -13621,6 +13650,15 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
continue;
if (!CountedExtracts.insert(EU.Scalar).second)
continue;
+ // The extract of the lane, which is free in the target, is not emitted.
+ Type *ScalarTy = EU.Scalar->getType();
+ if (HasFusedAlt && !MinBWs.contains(&EU.E) &&
+ VectorType::isValidElementType(ScalarTy) &&
+ TTI->getVectorInstrCost(
+ Instruction::ExtractElement,
+ getWidenedType(ScalarTy, EU.E.getVectorFactor()), CostKind,
+ EU.Lane) == TTI::TCC_Free)
+ continue;
++Total;
}
return Total;
@@ -17975,9 +18013,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// If this pattern is supported by the target then we consider the
// order.
if (TTIRef.isLegalAltInstr(cast<VectorType>(VecTy), Opcode0, Opcode1,
- OpcodeMask)) {
- InstructionCost AltVecCost = TTIRef.getAltInstrCost(
- cast<VectorType>(VecTy), Opcode0, Opcode1, OpcodeMask, CostKind);
+ OpcodeMask, E->Scalars)) {
+ InstructionCost AltVecCost =
+ TTIRef.getAltInstrCost(cast<VectorType>(VecTy), Opcode0, Opcode1,
+ OpcodeMask, CostKind, E->Scalars);
return AltVecCost < VecCost ? AltVecCost : VecCost;
}
// TODO: Check the reverse order too.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fused-alt-fmul.ll b/llvm/test/Transforms/SLPVectorizer/X86/fused-alt-fmul.ll
index d4eaae8327332..0d06cf3ab1947 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fused-alt-fmul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fused-alt-fmul.ll
@@ -7,29 +7,25 @@
define void @fmaddsub(ptr noalias %a0p, ptr noalias %a2p, ptr noalias %ep) {
; CHECK-LABEL: define void @fmaddsub(
; CHECK-SAME: ptr noalias [[A0P:%.*]], ptr noalias [[A2P:%.*]], ptr noalias [[EP:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[A0I_P:%.*]] = getelementptr inbounds i8, ptr [[A0P]], i64 8
; CHECK-NEXT: [[A2I_P:%.*]] = getelementptr inbounds i8, ptr [[A2P]], i64 8
-; CHECK-NEXT: [[EI_P:%.*]] = getelementptr inbounds i8, ptr [[EP]], i64 8
-; CHECK-NEXT: [[A0R:%.*]] = load double, ptr [[A0P]], align 8
-; CHECK-NEXT: [[A0I:%.*]] = load double, ptr [[A0I_P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A0P]], align 8
; CHECK-NEXT: [[A2R:%.*]] = load double, ptr [[A2P]], align 8
; CHECK-NEXT: [[A2I:%.*]] = load double, ptr [[A2I_P]], align 8
-; CHECK-NEXT: [[ER:%.*]] = load double, ptr [[EP]], align 8
-; CHECK-NEXT: [[EI:%.*]] = load double, ptr [[EI_P]], align 8
-; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[ER]], [[A2R]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[ER]], [[A2I]]
-; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[EI]], [[A2R]]
-; CHECK-NEXT: [[IM:%.*]] = fadd fast double [[M3]], [[M2]]
-; CHECK-NEXT: [[M4:%.*]] = fmul fast double [[EI]], [[A2I]]
-; CHECK-NEXT: [[RE:%.*]] = fsub fast double [[M1]], [[M4]]
-; CHECK-NEXT: [[HR:%.*]] = fadd fast double [[RE]], [[A0R]]
-; CHECK-NEXT: [[HI:%.*]] = fadd fast double [[IM]], [[A0I]]
-; CHECK-NEXT: [[SR:%.*]] = fsub fast double [[A0R]], [[RE]]
-; CHECK-NEXT: [[SI:%.*]] = fsub fast double [[A0I]], [[IM]]
-; CHECK-NEXT: store double [[SR]], ptr [[A2P]], align 8
-; CHECK-NEXT: store double [[SI]], ptr [[A2I_P]], align 8
-; CHECK-NEXT: store double [[HR]], ptr [[A0P]], align 8
-; CHECK-NEXT: store double [[HI]], ptr [[A0I_P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[EP]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[A2R]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[A2I]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = fsub fast <2 x double> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[TMP10]], <2 x double> [[TMP11]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP12]]
+; CHECK-NEXT: store <2 x double> [[TMP13]], ptr [[A2P]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <2 x double> [[TMP12]], [[TMP1]]
+; CHECK-NEXT: store <2 x double> [[TMP14]], ptr [[A0P]], align 8
; CHECK-NEXT: ret void
;
%a0i.p = getelementptr inbounds i8, ptr %a0p, i64 8
@@ -61,29 +57,25 @@ define void @fmaddsub(ptr noalias %a0p, ptr noalias %a2p, ptr noalias %ep) {
define void @fmsubadd(ptr noalias %a1p, ptr noalias %a3p, ptr noalias %ep) {
; CHECK-LABEL: define void @fmsubadd(
; CHECK-SAME: ptr noalias [[A1P:%.*]], ptr noalias [[A3P:%.*]], ptr noalias [[EP:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A1I_P:%.*]] = getelementptr inbounds i8, ptr [[A1P]], i64 8
-; CHECK-NEXT: [[A3I_P:%.*]] = getelementptr inbounds i8, ptr [[A3P]], i64 8
; CHECK-NEXT: [[EI_P:%.*]] = getelementptr inbounds i8, ptr [[EP]], i64 8
-; CHECK-NEXT: [[A1R:%.*]] = load double, ptr [[A1P]], align 8
-; CHECK-NEXT: [[A1I:%.*]] = load double, ptr [[A1I_P]], align 8
-; CHECK-NEXT: [[A3R:%.*]] = load double, ptr [[A3P]], align 8
-; CHECK-NEXT: [[A3I:%.*]] = load double, ptr [[A3I_P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A1P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[A3P]], align 8
; CHECK-NEXT: [[ER:%.*]] = load double, ptr [[EP]], align 8
; CHECK-NEXT: [[EI:%.*]] = load double, ptr [[EI_P]], align 8
-; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[A3R]], [[ER]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[A3I]], [[ER]]
-; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[A3R]], [[EI]]
-; CHECK-NEXT: [[IM:%.*]] = fsub fast double [[M2]], [[M3]]
-; CHECK-NEXT: [[M4:%.*]] = fmul fast double [[A3I]], [[EI]]
-; CHECK-NEXT: [[RE:%.*]] = fadd fast double [[M4]], [[M1]]
-; CHECK-NEXT: [[SR:%.*]] = fsub fast double [[A1R]], [[RE]]
-; CHECK-NEXT: [[SI:%.*]] = fsub fast double [[A1I]], [[IM]]
-; CHECK-NEXT: [[HR:%.*]] = fadd fast double [[RE]], [[A1R]]
-; CHECK-NEXT: [[HI:%.*]] = fadd fast double [[IM]], [[A1I]]
-; CHECK-NEXT: store double [[HR]], ptr [[A3P]], align 8
-; CHECK-NEXT: store double [[HI]], ptr [[A3I_P]], align 8
-; CHECK-NEXT: store double [[SR]], ptr [[A1P]], align 8
-; CHECK-NEXT: store double [[SI]], ptr [[A1I_P]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[ER]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[EI]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP2]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <2 x double> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[TMP10]], <2 x double> [[TMP11]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fadd fast <2 x double> [[TMP12]], [[TMP1]]
+; CHECK-NEXT: store <2 x double> [[TMP13]], ptr [[A3P]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP12]]
+; CHECK-NEXT: store <2 x double> [[TMP14]], ptr [[A1P]], align 8
; CHECK-NEXT: ret void
;
%a1i.p = getelementptr inbounds i8, ptr %a1p, i64 8
@@ -173,29 +165,25 @@ define void @no_contract(ptr noalias %a0p, ptr noalias %a2p, ptr noalias %ep) {
define void @fmaddsub_float(ptr noalias %a0p, ptr noalias %a2p, ptr noalias %ep) {
; CHECK-LABEL: define void @fmaddsub_float(
; CHECK-SAME: ptr noalias [[A0P:%.*]], ptr noalias [[A2P:%.*]], ptr noalias [[EP:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A0I_P:%.*]] = getelementptr inbounds i8, ptr [[A0P]], i64 4
; CHECK-NEXT: [[A2I_P:%.*]] = getelementptr inbounds i8, ptr [[A2P]], i64 4
-; CHECK-NEXT: [[EI_P:%.*]] = getelementptr inbounds i8, ptr [[EP]], i64 4
-; CHECK-NEXT: [[A0R:%.*]] = load float, ptr [[A0P]], align 4
-; CHECK-NEXT: [[A0I:%.*]] = load float, ptr [[A0I_P]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A0P]], align 4
; CHECK-NEXT: [[A2R:%.*]] = load float, ptr [[A2P]], align 4
; CHECK-NEXT: [[A2I:%.*]] = load float, ptr [[A2I_P]], align 4
-; CHECK-NEXT: [[ER:%.*]] = load float, ptr [[EP]], align 4
-; CHECK-NEXT: [[EI:%.*]] = load float, ptr [[EI_P]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul fast float [[ER]], [[A2R]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast float [[ER]], [[A2I]]
-; CHECK-NEXT: [[M3:%.*]] = fmul fast float [[EI]], [[A2R]]
-; CHECK-NEXT: [[IM:%.*]] = fadd fast float [[M3]], [[M2]]
-; CHECK-NEXT: [[M4:%.*]] = fmul fast float [[EI]], [[A2I]]
-; CHECK-NEXT: [[RE:%.*]] = fsub fast float [[M1]], [[M4]]
-; CHECK-NEXT: [[HR:%.*]] = fadd fast float [[RE]], [[A0R]]
-; CHECK-NEXT: [[HI:%.*]] = fadd fast float [[IM]], [[A0I]]
-; CHECK-NEXT: [[SR:%.*]] = fsub fast float [[A0R]], [[RE]]
-; CHECK-NEXT: [[SI:%.*]] = fsub fast float [[A0I]], [[IM]]
-; CHECK-NEXT: store float [[SR]], ptr [[A2P]], align 4
-; CHECK-NEXT: store float [[SI]], ptr [[A2I_P]], align 4
-; CHECK-NEXT: store float [[HR]], ptr [[A0P]], align 4
-; CHECK-NEXT: store float [[HI]], ptr [[A0I_P]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[EP]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[A2R]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A2I]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = fsub fast <2 x float> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd fast <2 x float> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP11]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fsub fast <2 x float> [[TMP1]], [[TMP12]]
+; CHECK-NEXT: store <2 x float> [[TMP13]], ptr [[A2P]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <2 x float> [[TMP12]], [[TMP1]]
+; CHECK-NEXT: store <2 x float> [[TMP14]], ptr [[A0P]], align 4
; CHECK-NEXT: ret void
;
%a0i.p = getelementptr inbounds i8, ptr %a0p, i64 4
@@ -227,29 +215,25 @@ define void @fmaddsub_float(ptr noalias %a0p, ptr noalias %a2p, ptr noalias %ep)
define void @fmsubadd_float(ptr noalias %a1p, ptr noalias %a3p, ptr noalias %ep) {
; CHECK-LABEL: define void @fmsubadd_float(
; CHECK-SAME: ptr noalias [[A1P:%.*]], ptr noalias [[A3P:%.*]], ptr noalias [[EP:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[A1I_P:%.*]] = getelementptr inbounds i8, ptr [[A1P]], i64 4
-; CHECK-NEXT: [[A3I_P:%.*]] = getelementptr inbounds i8, ptr [[A3P]], i64 4
; CHECK-NEXT: [[EI_P:%.*]] = getelementptr inbounds i8, ptr [[EP]], i64 4
-; CHECK-NEXT: [[A1R:%.*]] = load float, ptr [[A1P]], align 4
-; CHECK-NEXT: [[A1I:%.*]] = load float, ptr [[A1I_P]], align 4
-; CHECK-NEXT: [[A3R:%.*]] = load float, ptr [[A3P]], align 4
-; CHECK-NEXT: [[A3I:%.*]] = load float, ptr [[A3I_P]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A1P]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[A3P]], align 4
; CHECK-NEXT: [[ER:%.*]] = load float, ptr [[EP]], align 4
; CHECK-NEXT: [[EI:%.*]] = load float, ptr [[EI_P]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul fast float [[A3R]], [[ER]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast float [[A3I]], [[ER]]
-; CHECK-NEXT: [[M3:%.*]] = fmul fast float [[A3R]], [[EI]]
-; CHECK-NEXT: [[IM:%.*]] = fsub fast float [[M2]], [[M3]]
-; CHECK-NEXT: [[M4:%.*]] = fmul fast float [[A3I]], [[EI]]
-; CHECK-NEXT: [[RE:%.*]] = fadd fast float [[M4]], [[M1]]
-; CHECK-NEXT: [[SR:%.*]] = fsub fast float [[A1R]], [[RE]]
-; CHECK-NEXT: [[SI:%.*]] = fsub fast float [[A1I]], [[IM]]
-; CHECK-NEXT: [[HR:%.*]] = fadd fast float [[RE]], [[A1R]]
-; CHECK-NEXT: [[HI:%.*]] = fadd fast float [[IM]], [[A1I]]
-; CHECK-NEXT: store float [[HR]], ptr [[A3P]], align 4
-; CHECK-NEXT: store float [[HI]], ptr [[A3I_P]], align 4
-; CHECK-NEXT: store float [[SR]], ptr [[A1P]], align 4
-; CHECK-NEXT: store float [[SI]], ptr [[A1I_P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[ER]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[EI]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = fadd fast <2 x float> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <2 x float> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP11]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fadd fast <2 x float> [[TMP12]], [[TMP1]]
+; CHECK-NEXT: store <2 x float> [[TMP13]], ptr [[A3P]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = fsub fast <2 x float> [[TMP1]], [[TMP12]]
+; CHECK-NEXT: store <2 x float> [[TMP14]], ptr [[A1P]], align 4
; CHECK-NEXT: ret void
;
%a1i.p = getelementptr inbounds i8, ptr %a1p, i64 4
More information about the llvm-commits
mailing list