[llvm] f89baa6 - [SLP]Vectorize reductions with fptrunc/fpext round-trip links
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 05:23:02 PDT 2026
Author: Alexey Bataev
Date: 2026-08-13T08:22:57-04:00
New Revision: f89baa67381bb83d72f2f3d5063d91cc44b964e0
URL: https://github.com/llvm/llvm-project/commit/f89baa67381bb83d72f2f3d5063d91cc44b964e0
DIFF: https://github.com/llvm/llvm-project/commit/f89baa67381bb83d72f2f3d5063d91cc44b964e0.diff
LOG: [SLP]Vectorize reductions with fptrunc/fpext round-trip links
Look through the cast round-trips between reduction links: drop the
elidable ones, re-emit the others in the ordered emission.
Fixes #31425
Reviewers: bababuck, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/215571
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a3350cf3f3e52..85daf23009400 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29833,6 +29833,11 @@ class HorizontalReduction {
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
+ /// The narrowing cast of the cast round-trip following the reduction
+ /// operation that consumes the reduced value at the same index (in
+ /// collection order, before the reversal to accumulation order), or null.
+ /// Ordered reductions with cast-interleaved links only.
+ SmallVector<Instruction *> RoundedLinks;
/// The type of reduction operation.
RecurKind RdxKind;
/// Checks if the optimization of original scalar identity operations on
@@ -30254,11 +30259,14 @@ class HorizontalReduction {
ReducedVals.clear();
ReducedValsToOps.clear();
ReductionOps.clear();
+ RoundedLinks.clear();
RdxKind = getRdxKind(Root);
// Currently, only ordered fadd reductions are supported.
if (RdxKind != RecurKind::FAdd)
return false;
- if (isVectorizable(RdxKind, Root) != ReductionOrdering::Ordered)
+ // Reassociable chains with non-elidable cast round-trip links are
+ // handled here as well: the emission keeps the original order.
+ if (isVectorizable(RdxKind, Root) == ReductionOrdering::None)
return false;
// Ordered reductions only support simple binary ops, not min/max
@@ -30285,6 +30293,7 @@ class HorizontalReduction {
unsigned Depth = 0;
bool ChainComplete = false;
constexpr unsigned MaxReducedVals = 1024;
+ Instruction *PendingCast = nullptr;
while (TreeN) {
if (Depth++ > RecursionMaxDepth)
break;
@@ -30302,10 +30311,28 @@ class HorizontalReduction {
ReducedValsToOps[LeafVal].push_back(TreeN);
ReducedValsToOps[ChainVal].push_back(TreeN);
ReducedVals.back().push_back(LeafVal);
+ RoundedLinks.push_back(PendingCast);
+ PendingCast = nullptr;
auto *ChainInst = dyn_cast<Instruction>(ChainVal);
+ // The link may round the accumulator to a narrower type and extend it
+ // back; look through the cast pair, the rounding is re-emitted later.
+ if (ChainInst && getRdxKind(ChainInst) != RdxKind) {
+ if (Instruction *NarrowCast =
+ lookThroughCastRoundTrip(ChainVal, /*MustBeElidable=*/false)) {
+ auto *I = cast<Instruction>(NarrowCast->getOperand(0));
+ if (getRdxKind(I) == RdxKind &&
+ hasRequiredNumberOfUses(/*IsCmpSelMinMax=*/false, I)) {
+ ReductionOps[0].push_back(ChainVal);
+ ReductionOps[0].push_back(NarrowCast);
+ PendingCast = NarrowCast;
+ ChainInst = I;
+ }
+ }
+ }
if (!ChainInst || getRdxKind(ChainInst) != RdxKind ||
!hasRequiredNumberOfUses(/*IsCmpSelMinMax=*/false, ChainInst)) {
ReducedVals.back().push_back(ChainVal);
+ RoundedLinks.push_back(nullptr);
ChainComplete = true;
break;
}
@@ -30323,6 +30350,7 @@ class HorizontalReduction {
ReducedVals.pop_back();
ReducedValsToOps.clear();
ReductionOps.clear();
+ RoundedLinks.clear();
return false;
}
std::reverse(ReducedVals.back().begin(), ReducedVals.back().end());
@@ -30368,12 +30396,28 @@ class HorizontalReduction {
auto CheckOperands = [&](Instruction *TreeN,
SmallVectorImpl<Value *> &PossibleReducedVals,
SmallVectorImpl<Instruction *> &ReductionOps,
+ ReductionOpsType &AllReductionOps,
unsigned Level) {
for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
getNumberOfOperands(TreeN)))) {
Value *EdgeVal = getRdxOperand(TreeN, I);
- ReducedValsToOps[EdgeVal].push_back(TreeN);
auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
+ // The link may be an elidable cast round-trip; look through it to
+ // continue the chain. It is dropped when the reduction is folded.
+ if (EdgeInst && getRdxKind(EdgeInst) != RdxKind) {
+ if (Instruction *NarrowCast =
+ lookThroughCastRoundTrip(EdgeVal, /*MustBeElidable=*/true)) {
+ auto *SrcI = cast<Instruction>(NarrowCast->getOperand(0));
+ if (getRdxKind(SrcI) == RdxKind &&
+ hasRequiredNumberOfUses(IsCmpSelMinMax, SrcI)) {
+ AllReductionOps.push_back(EdgeVal);
+ AllReductionOps.push_back(NarrowCast);
+ EdgeVal = SrcI;
+ EdgeInst = SrcI;
+ }
+ }
+ }
+ ReducedValsToOps[EdgeVal].push_back(TreeN);
// If the edge is not an instruction, or it is
diff erent from the main
// reduction opcode or has too many uses - possible reduced value.
// Also, do not try to reduce const values, if the operation is not
@@ -30461,7 +30505,8 @@ class HorizontalReduction {
continue;
SmallVector<Value *> PossibleRedVals;
SmallVector<Instruction *> PossibleReductionOps;
- CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps, Level);
+ CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
+ ReductionOps[0], Level);
addReductionOps(TreeN);
ReducedValsCandidates.append(PossibleRedVals.begin(),
PossibleRedVals.end());
@@ -31557,35 +31602,71 @@ class HorizontalReduction {
// Fold leading scalars [0, SuccessStart) into an accumulator.
Type *DestTy = ReductionRoot->getType();
WeakTrackingVH VectorizedTree = nullptr;
- for (Value *RdxVal : ArrayRef(Candidates).take_front(SuccessStart)) {
+ if (!all_of(RoundedLinks, equal_to(nullptr))) {
+ // The round-trip links cannot be expressed by the reduction intrinsic:
+ // fold the values into a scalar chain in the original order, extracting
+ // the vectorized ones, and re-emit the round-trips between the folded
+ // operations. The emitted chain matches the same pattern again, so its
+ // operations are marked as analyzed to prevent repeated vectorization.
+ for (unsigned Idx : seq<unsigned>(N)) {
+ Value *RdxVal = Candidates[Idx];
+ Builder.SetCurrentDebugLocation(
+ ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
+ Value *NextVal = TrackedVals.at(RdxVal);
+ if (Idx >= SuccessStart && Idx < SuccessStart + SuccessWidth)
+ NextVal =
+ Builder.CreateExtractElement(SuccessRoot, Idx - SuccessStart);
+ if (!VectorizedTree) {
+ VectorizedTree = NextVal;
+ continue;
+ }
+ auto *Op = cast<Instruction>(createOp(Builder, RdxKind, VectorizedTree,
+ NextVal, "op.rdx", ReductionOps));
+ V.analyzedReductionRoot(Op);
+ VectorizedTree = Op;
+ Instruction *Narrow = RoundedLinks[N - 1 - Idx];
+ if (!Narrow)
+ continue;
+ Instruction *NewNarrow = Narrow->clone();
+ NewNarrow->setOperand(0, VectorizedTree);
+ Builder.Insert(NewNarrow);
+ Instruction *NewWide =
+ cast<Instruction>(*Narrow->user_begin())->clone();
+ NewWide->setOperand(0, NewNarrow);
+ VectorizedTree = Builder.Insert(NewWide);
+ }
+ } else {
+ for (Value *RdxVal : ArrayRef(Candidates).take_front(SuccessStart)) {
+ Builder.SetCurrentDebugLocation(
+ ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
+ if (!VectorizedTree)
+ VectorizedTree = TrackedVals.at(RdxVal);
+ else
+ VectorizedTree =
+ createOp(Builder, RdxKind, VectorizedTree, TrackedVals.at(RdxVal),
+ "op.rdx", ReductionOps);
+ }
+
+ // Emit ordered reduction for the vectorized window. The reduction only
+ // applies to floating point types.
+ assert(DestTy->isFPOrFPVectorTy() &&
+ SuccessRoot->getType()->isFPOrFPVectorTy() &&
+ "Expected floating point types for ordered reduction");
Builder.SetCurrentDebugLocation(
- ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
- if (!VectorizedTree)
- VectorizedTree = TrackedVals.at(RdxVal);
- else
+ cast<Instruction>(ReductionRoot)->getDebugLoc());
+ VectorizedTree = createSingleOp(Builder, *TTI, SuccessRoot, /*Scale=*/1,
+ /*IsSigned=*/false, DestTy,
+ /*ReducedInTree=*/false, VectorizedTree);
+
+ // Fold trailing scalars [SuccessStart+SuccessWidth, N).
+ for (Value *RdxVal :
+ ArrayRef(Candidates).drop_front(SuccessStart + SuccessWidth)) {
+ Builder.SetCurrentDebugLocation(
+ ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
VectorizedTree =
createOp(Builder, RdxKind, VectorizedTree, TrackedVals.at(RdxVal),
"op.rdx", ReductionOps);
- }
-
- // Emit ordered reduction for the vectorized window. The reduction only
- // applies to floating point types.
- assert(DestTy->isFPOrFPVectorTy() &&
- SuccessRoot->getType()->isFPOrFPVectorTy() &&
- "Expected floating point types for ordered reduction");
- Builder.SetCurrentDebugLocation(
- cast<Instruction>(ReductionRoot)->getDebugLoc());
- VectorizedTree = createSingleOp(Builder, *TTI, SuccessRoot, /*Scale=*/1,
- /*IsSigned=*/false, DestTy,
- /*ReducedInTree=*/false, VectorizedTree);
-
- // Fold trailing scalars [SuccessStart+SuccessWidth, N).
- for (Value *RdxVal :
- ArrayRef(Candidates).drop_front(SuccessStart + SuccessWidth)) {
- Builder.SetCurrentDebugLocation(
- ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
- VectorizedTree = createOp(Builder, RdxKind, VectorizedTree,
- TrackedVals.at(RdxVal), "op.rdx", ReductionOps);
+ }
}
ReductionRoot->replaceAllUsesWith(VectorizedTree);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 2889661006e1c..84e8cc2b66943 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -754,4 +754,20 @@ bool isOnceUsedSeed(const Instruction *I) {
I);
}
+Instruction *lookThroughCastRoundTrip(Value *V, bool MustBeElidable) {
+ auto *Wide = dyn_cast<FPExtInst>(V);
+ if (!Wide || !Wide->hasOneUse())
+ return nullptr;
+ auto *Narrow = dyn_cast<FPTruncInst>(Wide->getOperand(0));
+ if (!Narrow || !Narrow->hasOneUse())
+ return nullptr;
+ Value *Src = Narrow->getOperand(0);
+ if (!isa<Instruction>(Src) || Src->getType() != Wide->getType())
+ return nullptr;
+ if (MustBeElidable && !(Wide->hasAllowContract() && Wide->hasNoNaNs() &&
+ Wide->hasNoInfs() && Narrow->hasAllowContract()))
+ return nullptr;
+ return Narrow;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index d826e197656b1..bdc8881e14090 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -328,6 +328,14 @@ Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode);
/// dedicated attempt.
bool isOnceUsedSeed(const Instruction *I);
+/// If \p V is a single-use fpext of a single-use fptrunc forming a round-trip
+/// back to the type of \p V, returns the fptrunc; the round-trip source is its
+/// operand, always an instruction of the same type as \p V. If
+/// \p MustBeElidable, matches only when the intermediate rounding may be
+/// removed: both casts must allow contraction and the widening cast cannot
+/// produce nan/inf.
+Instruction *lookThroughCastRoundTrip(Value *V, bool MustBeElidable);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
index e2af602cc86cd..5c934d9e13087 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
@@ -117,84 +117,10 @@ entry:
define double @fadd_fptrunc_fpext_links_fast(ptr %x) {
; CHECK-LABEL: define double @fadd_fptrunc_fpext_links_fast(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fadd.v16f64(double 0.000000e+00, <16 x double> [[TMP1]])
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast double [[TMP2]], 1.000000e+00
-; CHECK-NEXT: [[T15:%.*]] = fptrunc fast double [[OP_RDX]] to float
-; CHECK-NEXT: [[E15:%.*]] = fpext fast float [[T15]] to double
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[A1:%.*]] = fadd fast double [[L1]], [[E15]]
-; CHECK-NEXT: [[T1:%.*]] = fptrunc fast double [[A1]] to float
-; CHECK-NEXT: [[E1:%.*]] = fpext fast float [[T1]] to double
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[A2:%.*]] = fadd fast double [[L2]], [[E1]]
-; CHECK-NEXT: [[T2:%.*]] = fptrunc fast double [[A2]] to float
-; CHECK-NEXT: [[E2:%.*]] = fpext fast float [[T2]] to double
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[L3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[A3:%.*]] = fadd fast double [[L3]], [[E2]]
-; CHECK-NEXT: [[T3:%.*]] = fptrunc fast double [[A3]] to float
-; CHECK-NEXT: [[E3:%.*]] = fpext fast float [[T3]] to double
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 32
-; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[P4]], align 8
-; CHECK-NEXT: [[A4:%.*]] = fadd fast double [[L4]], [[E3]]
-; CHECK-NEXT: [[T4:%.*]] = fptrunc fast double [[A4]] to float
-; CHECK-NEXT: [[E4:%.*]] = fpext fast float [[T4]] to double
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 40
-; CHECK-NEXT: [[L5:%.*]] = load double, ptr [[P5]], align 8
-; CHECK-NEXT: [[A5:%.*]] = fadd fast double [[L5]], [[E4]]
-; CHECK-NEXT: [[T5:%.*]] = fptrunc fast double [[A5]] to float
-; CHECK-NEXT: [[E5:%.*]] = fpext fast float [[T5]] to double
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 48
-; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[P6]], align 8
-; CHECK-NEXT: [[A6:%.*]] = fadd fast double [[L6]], [[E5]]
-; CHECK-NEXT: [[T6:%.*]] = fptrunc fast double [[A6]] to float
-; CHECK-NEXT: [[E6:%.*]] = fpext fast float [[T6]] to double
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
-; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
-; CHECK-NEXT: [[A7:%.*]] = fadd fast double [[L7]], [[E6]]
-; CHECK-NEXT: [[T7:%.*]] = fptrunc fast double [[A7]] to float
-; CHECK-NEXT: [[E7:%.*]] = fpext fast float [[T7]] to double
-; CHECK-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 64
-; CHECK-NEXT: [[L8:%.*]] = load double, ptr [[P8]], align 8
-; CHECK-NEXT: [[A8:%.*]] = fadd fast double [[L8]], [[E7]]
-; CHECK-NEXT: [[T8:%.*]] = fptrunc fast double [[A8]] to float
-; CHECK-NEXT: [[E8:%.*]] = fpext fast float [[T8]] to double
-; CHECK-NEXT: [[P9:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 72
-; CHECK-NEXT: [[L9:%.*]] = load double, ptr [[P9]], align 8
-; CHECK-NEXT: [[A9:%.*]] = fadd fast double [[L9]], [[E8]]
-; CHECK-NEXT: [[T9:%.*]] = fptrunc fast double [[A9]] to float
-; CHECK-NEXT: [[E9:%.*]] = fpext fast float [[T9]] to double
-; CHECK-NEXT: [[P10:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 80
-; CHECK-NEXT: [[L10:%.*]] = load double, ptr [[P10]], align 8
-; CHECK-NEXT: [[A10:%.*]] = fadd fast double [[L10]], [[E9]]
-; CHECK-NEXT: [[T10:%.*]] = fptrunc fast double [[A10]] to float
-; CHECK-NEXT: [[E10:%.*]] = fpext fast float [[T10]] to double
-; CHECK-NEXT: [[P11:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 88
-; CHECK-NEXT: [[L11:%.*]] = load double, ptr [[P11]], align 8
-; CHECK-NEXT: [[A11:%.*]] = fadd fast double [[L11]], [[E10]]
-; CHECK-NEXT: [[T11:%.*]] = fptrunc fast double [[A11]] to float
-; CHECK-NEXT: [[E11:%.*]] = fpext fast float [[T11]] to double
-; CHECK-NEXT: [[P12:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 96
-; CHECK-NEXT: [[L12:%.*]] = load double, ptr [[P12]], align 8
-; CHECK-NEXT: [[A12:%.*]] = fadd fast double [[L12]], [[E11]]
-; CHECK-NEXT: [[T12:%.*]] = fptrunc fast double [[A12]] to float
-; CHECK-NEXT: [[E12:%.*]] = fpext fast float [[T12]] to double
-; CHECK-NEXT: [[P13:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 104
-; CHECK-NEXT: [[L13:%.*]] = load double, ptr [[P13]], align 8
-; CHECK-NEXT: [[A13:%.*]] = fadd fast double [[L13]], [[E12]]
-; CHECK-NEXT: [[T13:%.*]] = fptrunc fast double [[A13]] to float
-; CHECK-NEXT: [[E13:%.*]] = fpext fast float [[T13]] to double
-; CHECK-NEXT: [[P14:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 112
-; CHECK-NEXT: [[L14:%.*]] = load double, ptr [[P14]], align 8
-; CHECK-NEXT: [[A14:%.*]] = fadd fast double [[L14]], [[E13]]
-; CHECK-NEXT: [[T14:%.*]] = fptrunc fast double [[A14]] to float
-; CHECK-NEXT: [[E14:%.*]] = fpext fast float [[T14]] to double
-; CHECK-NEXT: [[P15:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 120
-; CHECK-NEXT: [[L15:%.*]] = load double, ptr [[P15]], align 8
-; CHECK-NEXT: [[A15:%.*]] = fadd fast double [[L15]], [[E14]]
-; CHECK-NEXT: [[T16:%.*]] = fptrunc fast double [[A15]] to float
+; CHECK-NEXT: [[T16:%.*]] = fptrunc fast double [[OP_RDX]] to float
; CHECK-NEXT: [[E16:%.*]] = fpext fast float [[T16]] to double
; CHECK-NEXT: ret double [[E16]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
index a0702a186f41b..d56d4dbe4fa87 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
@@ -169,44 +169,49 @@ define double @ordered_fadd_fptrunc_fpext_links(ptr %x) {
; CHECK-LABEL: define double @ordered_fadd_fptrunc_fpext_links(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[A0:%.*]] = fadd double [[L0]], 1.000000e+00
-; CHECK-NEXT: [[T0:%.*]] = fptrunc double [[A0]] to float
-; CHECK-NEXT: [[TMP12:%.*]] = fpext float [[T0]] to double
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[P3]], align 8
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
+; CHECK-NEXT: [[L8:%.*]] = load double, ptr [[P7]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x double> <double 1.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison, double poison>, double [[L0]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP3]], <8 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x double> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x double> [[TMP7]], i64 1
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd double [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = fptrunc double [[OP_RDX1]] to float
-; CHECK-NEXT: [[TMP15:%.*]] = fpext float [[TMP13]] to double
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[P2]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = fpext float [[TMP13]] to double
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x double> [[TMP7]], i64 2
; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd double [[TMP14]], [[TMP15]]
; CHECK-NEXT: [[TMP16:%.*]] = fptrunc double [[OP_RDX2]] to float
-; CHECK-NEXT: [[TMP18:%.*]] = fpext float [[TMP16]] to double
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[P3]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = fpext float [[TMP16]] to double
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x double> [[TMP7]], i64 3
; CHECK-NEXT: [[OP_RDX3:%.*]] = fadd double [[TMP17]], [[TMP18]]
; CHECK-NEXT: [[TMP19:%.*]] = fptrunc double [[OP_RDX3]] to float
-; CHECK-NEXT: [[TMP21:%.*]] = fpext float [[TMP19]] to double
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 32
-; CHECK-NEXT: [[TMP20:%.*]] = load double, ptr [[P4]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = fpext float [[TMP19]] to double
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <8 x double> [[TMP7]], i64 4
; CHECK-NEXT: [[OP_RDX4:%.*]] = fadd double [[TMP20]], [[TMP21]]
; CHECK-NEXT: [[TMP22:%.*]] = fptrunc double [[OP_RDX4]] to float
-; CHECK-NEXT: [[TMP24:%.*]] = fpext float [[TMP22]] to double
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 40
-; CHECK-NEXT: [[TMP23:%.*]] = load double, ptr [[P5]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = fpext float [[TMP22]] to double
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x double> [[TMP7]], i64 5
; CHECK-NEXT: [[OP_RDX5:%.*]] = fadd double [[TMP23]], [[TMP24]]
; CHECK-NEXT: [[TMP25:%.*]] = fptrunc double [[OP_RDX5]] to float
-; CHECK-NEXT: [[TMP27:%.*]] = fpext float [[TMP25]] to double
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 48
-; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[P6]], align 8
+; CHECK-NEXT: [[TMP26:%.*]] = fpext float [[TMP25]] to double
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <8 x double> [[TMP7]], i64 6
; CHECK-NEXT: [[OP_RDX6:%.*]] = fadd double [[TMP26]], [[TMP27]]
; CHECK-NEXT: [[TMP28:%.*]] = fptrunc double [[OP_RDX6]] to float
-; CHECK-NEXT: [[L7:%.*]] = fpext float [[TMP28]] to double
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
-; CHECK-NEXT: [[TMP29:%.*]] = load double, ptr [[P7]], align 8
+; CHECK-NEXT: [[TMP29:%.*]] = fpext float [[TMP28]] to double
+; CHECK-NEXT: [[L7:%.*]] = extractelement <8 x double> [[TMP7]], i64 7
; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd double [[TMP29]], [[L7]]
; CHECK-NEXT: [[T7:%.*]] = fptrunc double [[OP_RDX7]] to float
-; CHECK-NEXT: [[E7:%.*]] = fpext float [[T7]] to double
+; CHECK-NEXT: [[TMP30:%.*]] = fpext float [[T7]] to double
+; CHECK-NEXT: [[OP_RDX8:%.*]] = fadd double [[TMP30]], [[L8]]
+; CHECK-NEXT: [[T8:%.*]] = fptrunc double [[OP_RDX8]] to float
+; CHECK-NEXT: [[E7:%.*]] = fpext float [[T8]] to double
; CHECK-NEXT: ret double [[E7]]
;
%l0 = load double, ptr %x, align 8
@@ -255,36 +260,41 @@ define double @ordered_fadd_mixed_links(ptr %x) {
; CHECK-LABEL: define double @ordered_fadd_mixed_links(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd double [[L0]], 1.000000e+00
-; CHECK-NEXT: [[TMP10:%.*]] = fptrunc double [[OP_RDX]] to float
-; CHECK-NEXT: [[TMP11:%.*]] = fpext float [[TMP10]] to double
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[P3]], align 8
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
+; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x double> <double 1.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison, double poison>, double [[L0]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP3]], <8 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[L1:%.*]] = extractelement <8 x double> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x double> [[TMP7]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = fadd double [[L1]], [[TMP11]]
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[OP_RDX1:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd double [[OP_RDX1]], [[TMP13]]
+; CHECK-NEXT: [[TMP10:%.*]] = fptrunc double [[TMP13]] to float
+; CHECK-NEXT: [[TMP24:%.*]] = fpext float [[TMP10]] to double
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x double> [[TMP7]], i64 2
+; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd double [[TMP24]], [[TMP12]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <8 x double> [[TMP7]], i64 3
+; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd double [[OP_RDX1]], [[TMP25]]
; CHECK-NEXT: [[TMP14:%.*]] = fptrunc double [[OP_RDX2]] to float
; CHECK-NEXT: [[TMP15:%.*]] = fpext float [[TMP14]] to double
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[L3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = fadd double [[L3]], [[TMP15]]
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 32
-; CHECK-NEXT: [[OP_RDX3:%.*]] = load double, ptr [[P4]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x double> [[TMP7]], i64 4
+; CHECK-NEXT: [[OP_RDX3:%.*]] = fadd double [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x double> [[TMP7]], i64 5
; CHECK-NEXT: [[OP_RDX4:%.*]] = fadd double [[OP_RDX3]], [[TMP17]]
; CHECK-NEXT: [[TMP18:%.*]] = fptrunc double [[OP_RDX4]] to float
; CHECK-NEXT: [[TMP19:%.*]] = fpext float [[TMP18]] to double
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 40
-; CHECK-NEXT: [[L5:%.*]] = load double, ptr [[P5]], align 8
-; CHECK-NEXT: [[TMP21:%.*]] = fadd double [[L5]], [[TMP19]]
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 48
-; CHECK-NEXT: [[OP_RDX5:%.*]] = load double, ptr [[P6]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x double> [[TMP7]], i64 6
+; CHECK-NEXT: [[OP_RDX5:%.*]] = fadd double [[TMP19]], [[TMP20]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <8 x double> [[TMP7]], i64 7
; CHECK-NEXT: [[OP_RDX6:%.*]] = fadd double [[OP_RDX5]], [[TMP21]]
; CHECK-NEXT: [[TMP22:%.*]] = fptrunc double [[OP_RDX6]] to float
; CHECK-NEXT: [[TMP23:%.*]] = fpext float [[TMP22]] to double
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
-; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
-; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd double [[L7]], [[TMP23]]
+; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd double [[TMP23]], [[L7]]
; CHECK-NEXT: ret double [[OP_RDX7]]
;
%l0 = load double, ptr %x, align 8
@@ -325,45 +335,50 @@ define double @ordered_fadd_reassoc_nonelidable_casts(ptr %x) {
; CHECK-LABEL: define double @ordered_fadd_reassoc_nonelidable_casts(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast double [[L0]], 1.000000e+00
-; CHECK-NEXT: [[TMP10:%.*]] = fptrunc nnan ninf double [[OP_RDX]] to float
-; CHECK-NEXT: [[TMP11:%.*]] = fpext nnan ninf float [[TMP10]] to double
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast double [[L1]], [[TMP11]]
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[P3]], align 8
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
+; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x double> <double 1.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison, double poison>, double [[L0]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP3]], <8 x double> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP7]], i64 1
+; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd nnan ninf double [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[TMP13:%.*]] = fptrunc nnan ninf double [[OP_RDX1]] to float
; CHECK-NEXT: [[TMP14:%.*]] = fpext nnan ninf float [[TMP13]] to double
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd fast double [[L2]], [[TMP14]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x double> [[TMP7]], i64 2
+; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd nnan ninf double [[TMP14]], [[TMP12]]
; CHECK-NEXT: [[TMP16:%.*]] = fptrunc nnan ninf double [[OP_RDX2]] to float
; CHECK-NEXT: [[TMP17:%.*]] = fpext nnan ninf float [[TMP16]] to double
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[L3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[OP_RDX3:%.*]] = fadd fast double [[L3]], [[TMP17]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x double> [[TMP7]], i64 3
+; CHECK-NEXT: [[OP_RDX3:%.*]] = fadd nnan ninf double [[TMP17]], [[TMP15]]
; CHECK-NEXT: [[TMP19:%.*]] = fptrunc nnan ninf double [[OP_RDX3]] to float
; CHECK-NEXT: [[TMP20:%.*]] = fpext nnan ninf float [[TMP19]] to double
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 32
-; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[P4]], align 8
-; CHECK-NEXT: [[OP_RDX4:%.*]] = fadd fast double [[L4]], [[TMP20]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x double> [[TMP7]], i64 4
+; CHECK-NEXT: [[OP_RDX4:%.*]] = fadd nnan ninf double [[TMP20]], [[TMP18]]
; CHECK-NEXT: [[TMP22:%.*]] = fptrunc nnan ninf double [[OP_RDX4]] to float
; CHECK-NEXT: [[TMP23:%.*]] = fpext nnan ninf float [[TMP22]] to double
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 40
-; CHECK-NEXT: [[L5:%.*]] = load double, ptr [[P5]], align 8
-; CHECK-NEXT: [[OP_RDX5:%.*]] = fadd fast double [[L5]], [[TMP23]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <8 x double> [[TMP7]], i64 5
+; CHECK-NEXT: [[OP_RDX5:%.*]] = fadd nnan ninf double [[TMP23]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = fptrunc nnan ninf double [[OP_RDX5]] to float
; CHECK-NEXT: [[TMP26:%.*]] = fpext nnan ninf float [[TMP25]] to double
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 48
-; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[P6]], align 8
-; CHECK-NEXT: [[OP_RDX6:%.*]] = fadd fast double [[L6]], [[TMP26]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x double> [[TMP7]], i64 6
+; CHECK-NEXT: [[OP_RDX6:%.*]] = fadd nnan ninf double [[TMP26]], [[TMP24]]
; CHECK-NEXT: [[TMP28:%.*]] = fptrunc nnan ninf double [[OP_RDX6]] to float
; CHECK-NEXT: [[TMP29:%.*]] = fpext nnan ninf float [[TMP28]] to double
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 56
-; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
-; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd fast double [[L7]], [[TMP29]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <8 x double> [[TMP7]], i64 7
+; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd nnan ninf double [[TMP29]], [[TMP27]]
; CHECK-NEXT: [[T7:%.*]] = fptrunc nnan ninf double [[OP_RDX7]] to float
; CHECK-NEXT: [[E7:%.*]] = fpext nnan ninf float [[T7]] to double
-; CHECK-NEXT: ret double [[E7]]
+; CHECK-NEXT: [[OP_RDX8:%.*]] = fadd nnan ninf double [[E7]], [[L7]]
+; CHECK-NEXT: [[T8:%.*]] = fptrunc nnan ninf double [[OP_RDX8]] to float
+; CHECK-NEXT: [[E8:%.*]] = fpext nnan ninf float [[T8]] to double
+; CHECK-NEXT: ret double [[E8]]
;
%l0 = load double, ptr %x, align 8
%a0 = fadd fast double %l0, 1.000000e+00
More information about the llvm-commits
mailing list