[llvm] [SLP] Fix canConvertToFMA fmul costing (PR #216425)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 15:55:41 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/216425
>From 554fc8e30174ad10479dfcff6859da0b4f2f83b0 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 14 Aug 2026 12:51:56 +0200
Subject: [PATCH 01/13] [SLP] Fix canConvertToFMA operand selection and fmul
costing
canConvertToFMA only looked for the fmul on operand 0 of the fadd, so the
accumulator shape fadd acc, a * b was never recognized. Check both operands
for chains that do not allow reassociation.
Also price the unfused fmul without a context instruction. Targets that
model fma fusion price a contractable fmul as free, which discounted the
scalar side of the comparison too and fmuladd never looked profitable.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 49 +++++--
.../AMDGPU/elementwise-fma-operand1.ll | 130 ++++++++++++++++++
2 files changed, 169 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53816d49de722..6e5628104f0f9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14382,18 +14382,44 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
- InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
- if (!OpS.valid())
- return InstructionCost::getInvalid();
-
- if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
- return InstructionCost::getInvalid();
- if (!CheckForContractable(Operands.front()))
+ // The fmul may sit on either side of the add/sub. Look past operand 0 only
+ // for chains that do not allow reassociation. A reassociative chain can be
+ // vectorized into a vector fmul feeding a reduction, which is usually
+ // better than the scalar fma chain this check protects.
+ bool AllowReassoc = any_of(VL, [](Value *V) {
+ auto *FPCI = dyn_cast<FPMathOperator>(V);
+ return FPCI && FPCI->getFastMathFlags().allowReassoc();
+ });
+ auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
+ for (unsigned Idx : seq<unsigned>(0, AllowReassoc ? 1 : Operands.size())) {
+ InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
+ if (!CandS.valid() || CandS.isAltShuffle() ||
+ CandS.getOpcode() != Instruction::FMul)
+ continue;
+ if (!CheckForContractable(Operands[Idx]))
+ continue;
+ return Idx;
+ }
+ return std::nullopt;
+ };
+ std::optional<unsigned> FMulIdx = GetFMulOperandIdx();
+ if (!FMulIdx)
return InstructionCost::getInvalid();
+ InstructionsState OpS = getSameOpcode(Operands[*FMulIdx], TLI);
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ // Price the fmul as not fused with its user. Passing a context instruction
+ // would let targets that model the fusion discount the unfused side of the
+ // comparison as well.
+ auto GetUnfusedFMulCost = [&](Instruction *I) {
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(Instruction::FMul, I->getType(), CostKind,
+ Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
+ };
FastMathFlags FMF;
FMF.set();
for (Value *V : VL) {
@@ -14406,7 +14432,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
}
unsigned NumOps = 0;
- for (auto [V, Op] : zip(VL, Operands.front())) {
+ for (auto [V, Op] : zip(VL, Operands[*FMulIdx])) {
if (S.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(Op);
@@ -14420,7 +14446,9 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ FMulPlusFAddCost += I->getOpcode() == Instruction::FMul
+ ? GetUnfusedFMulCost(I)
+ : TTI.getInstructionCost(I, CostKind);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
@@ -15225,7 +15253,8 @@ void BoUpSLP::transformNodes() {
break;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
- TreeEntry *FMulEntry = getOperandEntry(&E, 0);
+ TreeEntry *FMulEntry =
+ getOperandEntry(&E, IsOneUseVectorFMulOperand(LHS) ? 0 : 1);
if (FMulEntry->UserTreeIndex &&
FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
new file mode 100644
index 0000000000000..efa8f5c4dd930
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
+
+; Elementwise d = c + a * b where the fmul is operand 1 of the fadd. Marking
+; a hardcoded operand 0 turned the c load bundle into a CombinedVectorize
+; node and made vectorizeTree abort.
+
+define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @axpy4_contract(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP5]], [[TMP8]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %c0 = load float, ptr %c, align 4
+ %a0 = load float, ptr %a, align 4
+ %b0 = load float, ptr %b, align 4
+ %m0 = fmul contract float %a0, %b0
+ %r0 = fadd contract float %c0, %m0
+ store float %r0, ptr %d, align 4
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %c1 = load float, ptr %cp1, align 4
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %a1 = load float, ptr %ap1, align 4
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %b1 = load float, ptr %bp1, align 4
+ %m1 = fmul contract float %a1, %b1
+ %r1 = fadd contract float %c1, %m1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ store float %r1, ptr %dp1, align 4
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %c2 = load float, ptr %cp2, align 4
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %a2 = load float, ptr %ap2, align 4
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %b2 = load float, ptr %bp2, align 4
+ %m2 = fmul contract float %a2, %b2
+ %r2 = fadd contract float %c2, %m2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ store float %r2, ptr %dp2, align 4
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %c3 = load float, ptr %cp3, align 4
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %a3 = load float, ptr %ap3, align 4
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %b3 = load float, ptr %bp3, align 4
+ %m3 = fmul contract float %a3, %b3
+ %r3 = fadd contract float %c3, %m3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ store float %r3, ptr %dp3, align 4
+ ret void
+}
+
+define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @axpy4_reassoc(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc contract <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc contract <2 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul reassoc contract <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd reassoc contract <2 x float> [[TMP5]], [[TMP8]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %c0 = load float, ptr %c, align 4
+ %a0 = load float, ptr %a, align 4
+ %b0 = load float, ptr %b, align 4
+ %m0 = fmul contract reassoc float %a0, %b0
+ %r0 = fadd contract reassoc float %c0, %m0
+ store float %r0, ptr %d, align 4
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %c1 = load float, ptr %cp1, align 4
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %a1 = load float, ptr %ap1, align 4
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %b1 = load float, ptr %bp1, align 4
+ %m1 = fmul contract reassoc float %a1, %b1
+ %r1 = fadd contract reassoc float %c1, %m1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ store float %r1, ptr %dp1, align 4
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %c2 = load float, ptr %cp2, align 4
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %a2 = load float, ptr %ap2, align 4
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %b2 = load float, ptr %bp2, align 4
+ %m2 = fmul contract reassoc float %a2, %b2
+ %r2 = fadd contract reassoc float %c2, %m2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ store float %r2, ptr %dp2, align 4
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %c3 = load float, ptr %cp3, align 4
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %a3 = load float, ptr %ap3, align 4
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %b3 = load float, ptr %bp3, align 4
+ %m3 = fmul contract reassoc float %a3, %b3
+ %r3 = fadd contract reassoc float %c3, %m3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ store float %r3, ptr %dp3, align 4
+ ret void
+}
>From 6f88c8a36a58695280f6390846e80bbbbf580c13 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 15 Aug 2026 16:48:53 +0200
Subject: [PATCH 02/13] apply comments
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6e5628104f0f9..67c52d4e8cc52 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14386,10 +14386,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// for chains that do not allow reassociation. A reassociative chain can be
// vectorized into a vector fmul feeding a reduction, which is usually
// better than the scalar fma chain this check protects.
- bool AllowReassoc = any_of(VL, [](Value *V) {
- auto *FPCI = dyn_cast<FPMathOperator>(V);
- return FPCI && FPCI->getFastMathFlags().allowReassoc();
- });
+ bool AllowReassoc = any_of(
+ VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
for (unsigned Idx : seq<unsigned>(0, AllowReassoc ? 1 : Operands.size())) {
InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
@@ -14414,6 +14412,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// would let targets that model the fusion discount the unfused side of the
// comparison as well.
auto GetUnfusedFMulCost = [&](Instruction *I) {
+ assert(I->getOpcode() == Instruction::FMul && "Expected an fmul");
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
return TTI.getArithmeticInstrCost(Instruction::FMul, I->getType(), CostKind,
@@ -14446,9 +14445,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += I->getOpcode() == Instruction::FMul
- ? GetUnfusedFMulCost(I)
- : TTI.getInstructionCost(I, CostKind);
+ FMulPlusFAddCost += GetUnfusedFMulCost(I);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
>From 7f9561465cc4f8f72b5723d2fa201b4300ba245b Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 15 Aug 2026 23:28:39 +0200
Subject: [PATCH 03/13] extra fixes
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 15 +++++---
.../AMDGPU/elementwise-fma-operand1.ll | 36 ++++++++++++++++---
2 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 67c52d4e8cc52..b74dd7f80f5b8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14383,13 +14383,17 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
// The fmul may sit on either side of the add/sub. Look past operand 0 only
- // for chains that do not allow reassociation. A reassociative chain can be
- // vectorized into a vector fmul feeding a reduction, which is usually
- // better than the scalar fma chain this check protects.
+ // for chains that do not allow reassociation. The gate is profitability, not
+ // correctness. A reassociative chain can be vectorized into a vector fmul
+ // feeding a reduction, which is usually better than the scalar fma chain
+ // this check protects. An fsub can only fold an fmul on its left, so stop at
+ // operand 0 there as well.
bool AllowReassoc = any_of(
VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
+ bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
- for (unsigned Idx : seq<unsigned>(0, AllowReassoc ? 1 : Operands.size())) {
+ for (unsigned Idx :
+ seq<unsigned>(0, OnlyFirstOperand ? 1 : Operands.size())) {
InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
if (!CandS.valid() || CandS.isAltShuffle() ||
CandS.getOpcode() != Instruction::FMul)
@@ -14428,7 +14432,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (!S.isCopyableElement(I))
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ FMulPlusFAddCost +=
+ TTI.getArithmeticInstrCost(S.getOpcode(), I->getType(), CostKind);
}
unsigned NumOps = 0;
for (auto [V, Op] : zip(VL, Operands[*FMulIdx])) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
index 44f55e536179a..b43c0a8fdf91c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -2,12 +2,16 @@
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=14 < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -slp-threshold=14 < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -slp-threshold=14 < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12
-; Elementwise d = c + a * b, where the fmul is operand 1 of the fadd. The
-; threshold puts the decision right at the cost boundary, so how the fmul and
-; fadd are priced against a fused fma is what decides it. These targets halve
-; the cost of a packed fmul, which is what tempts SLP into vectorizing and
-; breaking the scalar fma chain.
+; Elementwise d = c + a * b, where the fmul is operand 1 of the fadd. These
+; targets halve the cost of a packed fmul, so SLP is tempted to vectorize and
+; break the scalar fma chain. The 14 runs sit at the cost boundary. The 12 runs
+; vectorize either way and guard against the fmuladd marking landing on the load
+; at operand 0 after the fma detection picked the fmul at operand 1, which
+; asserts.
define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @axpy4_contract(
@@ -51,6 +55,17 @@ define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr
; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
; CHECK-NEXT: ret void
;
+; THR12-LABEL: define void @axpy4_contract(
+; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; THR12-NEXT: [[ENTRY:.*:]]
+; THR12-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; THR12-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; THR12-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; THR12-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; THR12-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; THR12-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; THR12-NEXT: ret void
+;
entry:
%c0 = load float, ptr %c, align 4
%a0 = load float, ptr %a, align 4
@@ -103,6 +118,17 @@ define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr n
; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
+; THR12-LABEL: define void @axpy4_reassoc(
+; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; THR12-NEXT: [[ENTRY:.*:]]
+; THR12-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; THR12-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; THR12-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; THR12-NEXT: [[TMP3:%.*]] = fmul reassoc contract <4 x float> [[TMP1]], [[TMP2]]
+; THR12-NEXT: [[TMP4:%.*]] = fadd reassoc contract <4 x float> [[TMP0]], [[TMP3]]
+; THR12-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; THR12-NEXT: ret void
+;
entry:
%c0 = load float, ptr %c, align 4
%a0 = load float, ptr %a, align 4
>From 325b41fcbad6bbc3a1f1d3880d9498fcf717890f Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 16 Aug 2026 11:36:35 +0200
Subject: [PATCH 04/13] format
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b74dd7f80f5b8..fcf526bfecb75 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14388,8 +14388,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// feeding a reduction, which is usually better than the scalar fma chain
// this check protects. An fsub can only fold an fmul on its left, so stop at
// operand 0 there as well.
- bool AllowReassoc = any_of(
- VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
+ bool AllowReassoc =
+ any_of(VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
for (unsigned Idx :
>From f57929be7204d307d599b151f439bd5b269096c7 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 16 Aug 2026 16:55:28 +0200
Subject: [PATCH 05/13] Require the whole bundle to be reassoc, and take the
operand count from the helper
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fcf526bfecb75..0432a9deac6ea 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14389,11 +14389,13 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// this check protects. An fsub can only fold an fmul on its left, so stop at
// operand 0 there as well.
bool AllowReassoc =
- any_of(VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
+ all_of(VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
+ unsigned NumCandidateOps =
+ OnlyFirstOperand ? 1
+ : getNumberOfPotentiallyCommutativeOps(S.getMainOp());
auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
- for (unsigned Idx :
- seq<unsigned>(0, OnlyFirstOperand ? 1 : Operands.size())) {
+ for (unsigned Idx : seq<unsigned>(0, NumCandidateOps)) {
InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
if (!CandS.valid() || CandS.isAltShuffle() ||
CandS.getOpcode() != Instruction::FMul)
>From 7221528e02cf4a6322cec42297137c21aac54126 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 16 Aug 2026 18:22:10 +0200
Subject: [PATCH 06/13] Cover the mixed-reassoc bundle the all_of change fixes
---
.../AMDGPU/elementwise-fma-operand1.ll | 126 +++++++++++++++++-
1 file changed, 125 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
index b43c0a8fdf91c..eed8216c5b1d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -11,7 +11,8 @@
; break the scalar fma chain. The 14 runs sit at the cost boundary. The 12 runs
; vectorize either way and guard against the fmuladd marking landing on the load
; at operand 0 after the fma detection picked the fmul at operand 1, which
-; asserts.
+; asserts. axpy4_mixed_reassoc carries reassoc on one lane only, so the whole
+; bundle has to be reassociative before the search gives up on operand 1.
define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @axpy4_contract(
@@ -168,3 +169,126 @@ entry:
store float %r3, ptr %dp3, align 4
ret void
}
+
+define void @axpy4_mixed_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @axpy4_mixed_reassoc(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
+; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
+; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
+; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: ret void
+;
+; THR12-LABEL: define void @axpy4_mixed_reassoc(
+; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; THR12-NEXT: [[ENTRY:.*:]]
+; THR12-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; THR12-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; THR12-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; THR12-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; THR12-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
+; THR12-NEXT: store float [[R0]], ptr [[D]], align 4
+; THR12-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; THR12-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; THR12-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; THR12-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; THR12-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; THR12-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; THR12-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; THR12-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
+; THR12-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; THR12-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; THR12-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; THR12-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; THR12-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; THR12-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; THR12-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; THR12-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; THR12-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; THR12-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
+; THR12-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; THR12-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; THR12-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; THR12-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; THR12-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; THR12-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; THR12-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; THR12-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; THR12-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; THR12-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
+; THR12-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; THR12-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; THR12-NEXT: ret void
+;
+entry:
+ %c0 = load float, ptr %c, align 4
+ %a0 = load float, ptr %a, align 4
+ %b0 = load float, ptr %b, align 4
+ %m0 = fmul contract float %a0, %b0
+ %r0 = fadd contract reassoc float %c0, %m0
+ store float %r0, ptr %d, align 4
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %c1 = load float, ptr %cp1, align 4
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %a1 = load float, ptr %ap1, align 4
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %b1 = load float, ptr %bp1, align 4
+ %m1 = fmul contract float %a1, %b1
+ %r1 = fadd contract float %c1, %m1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ store float %r1, ptr %dp1, align 4
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %c2 = load float, ptr %cp2, align 4
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %a2 = load float, ptr %ap2, align 4
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %b2 = load float, ptr %bp2, align 4
+ %m2 = fmul contract float %a2, %b2
+ %r2 = fadd contract float %c2, %m2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ store float %r2, ptr %dp2, align 4
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %c3 = load float, ptr %cp3, align 4
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %a3 = load float, ptr %ap3, align 4
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %b3 = load float, ptr %bp3, align 4
+ %m3 = fmul contract float %a3, %b3
+ %r3 = fadd contract float %c3, %m3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ store float %r3, ptr %dp3, align 4
+ ret void
+}
>From 8dfbc8435bfa00486938f82f3ce61df2b727df72 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 17 Aug 2026 01:17:12 +0200
Subject: [PATCH 07/13] Skip undef and copyable lanes in the reassoc check
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e575918bf582c..bedea0c5ad873 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14429,8 +14429,12 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// feeding a reduction, which is usually better than the scalar fma chain
// this check protects. An fsub can only fold an fmul on its left, so stop at
// operand 0 there as well.
- bool AllowReassoc =
- all_of(VL, [](Value *V) { return match(V, m_AllowReassoc(m_Value())); });
+ bool AllowReassoc = all_of(VL, [&](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || S.isCopyableElement(I))
+ return true;
+ return match(I, m_AllowReassoc(m_Value()));
+ });
bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
unsigned NumCandidateOps =
OnlyFirstOperand ? 1
>From 796c8dddef5b7bf1a4e49b3d732addb1fabf4af4 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 17 Aug 2026 23:58:31 +0200
Subject: [PATCH 08/13] showcase for revert AllowReassoc
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 9 +-
.../AMDGPU/elementwise-fma-operand1.ll | 168 ++++++------------
.../X86/horizontal-fadd-with-sub.ll | 34 ++--
.../X86/redux-feed-buildvector.ll | 88 +++++++--
.../X86/select-logical-or-and-i1-vector.ll | 102 ++++++-----
5 files changed, 208 insertions(+), 193 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 28699b2664387..4c0784e20c48c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14425,18 +14425,15 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
// The fmul may sit on either side of the add/sub. Look past operand 0 only
- // for chains that do not allow reassociation. The gate is profitability, not
- // correctness. A reassociative chain can be vectorized into a vector fmul
- // feeding a reduction, which is usually better than the scalar fma chain
- // this check protects. An fsub can only fold an fmul on its left, so stop at
- // operand 0 there as well.
+ // for chains that allow reassociation. An fsub can only fold an fmul on its
+ // left, so stop at operand 0 there as well.
bool AllowReassoc = all_of(VL, [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
if (!I || S.isCopyableElement(I))
return true;
return match(I, m_AllowReassoc(m_Value()));
});
- bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
+ bool OnlyFirstOperand = !AllowReassoc || S.getOpcode() == Instruction::FSub;
unsigned NumCandidateOps =
OnlyFirstOperand ? 1
: getNumberOfPotentiallyCommutativeOps(S.getMainOp());
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
index eed8216c5b1d2..daee269eef852 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -18,42 +18,12 @@ define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr
; CHECK-LABEL: define void @axpy4_contract(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[R0:%.*]] = fadd contract float [[C0]], [[M0]]
-; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
-; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_contract(
@@ -111,12 +81,42 @@ define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr n
; CHECK-LABEL: define void @axpy4_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc contract <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc contract <4 x float> [[TMP0]], [[TMP3]]
-; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul reassoc contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
+; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; CHECK-NEXT: [[M1:%.*]] = fmul reassoc contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[R1:%.*]] = fadd reassoc contract float [[C1]], [[M1]]
+; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; CHECK-NEXT: [[M2:%.*]] = fmul reassoc contract float [[A2]], [[B2]]
+; CHECK-NEXT: [[R2:%.*]] = fadd reassoc contract float [[C2]], [[M2]]
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; CHECK-NEXT: [[M3:%.*]] = fmul reassoc contract float [[A3]], [[B3]]
+; CHECK-NEXT: [[R3:%.*]] = fadd reassoc contract float [[C3]], [[M3]]
+; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_reassoc(
@@ -174,83 +174,23 @@ define void @axpy4_mixed_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b,
; CHECK-LABEL: define void @axpy4_mixed_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
-; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
-; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_mixed_reassoc(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR12-NEXT: [[ENTRY:.*:]]
-; THR12-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; THR12-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; THR12-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; THR12-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; THR12-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
-; THR12-NEXT: store float [[R0]], ptr [[D]], align 4
-; THR12-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; THR12-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; THR12-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; THR12-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; THR12-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; THR12-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; THR12-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; THR12-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; THR12-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; THR12-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; THR12-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; THR12-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; THR12-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; THR12-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; THR12-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; THR12-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; THR12-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; THR12-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; THR12-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; THR12-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; THR12-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; THR12-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; THR12-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; THR12-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; THR12-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; THR12-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; THR12-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; THR12-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; THR12-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; THR12-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; THR12-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; THR12-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; THR12-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; THR12-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; THR12-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; THR12-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; THR12-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
index b1b163e64dc79..96dbc39a5b780 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
@@ -9,16 +9,19 @@ define double @fsub_fmul_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_fmul_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
+; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
+; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
+; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
+; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fmul reassoc nsz contract double [[Y1]], [[X1]]
; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
; CHECK-NEXT: [[ZSUM:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc nsz contract double [[TMP5]], [[ZSUM]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc nsz contract double [[SUB]], [[TMP4]]
; CHECK-NEXT: ret double [[TMP6]]
;
@@ -46,25 +49,28 @@ define double @fsub_fmul_4(ptr %x, ptr %y, ptr %z) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
+; CHECK-NEXT: [[X16:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 16
+; CHECK-NEXT: [[Y16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 16
; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 24
; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 24
; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
+; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X8]], align 8
+; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y8]], align 8
; CHECK-NEXT: [[MUL16:%.*]] = fmul reassoc nsz contract double [[Y3]], [[X3]]
+; CHECK-NEXT: [[X2:%.*]] = load double, ptr [[X16]], align 8
+; CHECK-NEXT: [[Y2:%.*]] = load double, ptr [[Y16]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = fmul reassoc nsz contract double [[Y2]], [[X2]]
+; CHECK-NEXT: [[X4:%.*]] = load double, ptr [[X24]], align 8
+; CHECK-NEXT: [[Y4:%.*]] = load double, ptr [[Y24]], align 8
+; CHECK-NEXT: [[MUL17:%.*]] = fmul reassoc nsz contract double [[Y4]], [[X4]]
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[Z]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[MUL]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[MUL]], [[MUL16]]
; CHECK-NEXT: [[T3:%.*]] = fadd reassoc nsz contract double [[T1]], [[TMP7]]
; CHECK-NEXT: [[TMP6:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
; CHECK-NEXT: [[ADD13:%.*]] = fsub reassoc nsz contract double [[T3]], [[TMP6]]
-; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[ADD13]], [[MUL16]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[ADD13]], [[MUL17]]
; CHECK-NEXT: ret double [[TMP5]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
index 67054d202d5a2..1801c14a72982 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
@@ -10,17 +10,83 @@ declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32 immarg
define void @test(ptr nocapture readonly %arg, ptr nocapture readonly %arg1, ptr nocapture %arg2) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x ptr> poison, ptr [[ARG:%.*]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x ptr> [[TMP0]], <8 x ptr> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP1]], <8 x i64> <i64 1, i64 3, i64 5, i64 7, i64 9, i64 11, i64 13, i64 15>
-; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1:%.*]], i64 16
-; CHECK-NEXT: [[TMP2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP5]], <8 x i1> splat (i1 true), <8 x double> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[GEP2_0]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <8 x double> [[TMP3]], [[TMP2]]
-; CHECK-NEXT: [[TMP6:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x double> [[TMP6]], [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
-; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
+; CHECK-NEXT: [[GEP1_0:%.*]] = getelementptr inbounds double, ptr [[ARG:%.*]], i64 1
+; CHECK-NEXT: [[LD1_0:%.*]] = load double, ptr [[GEP1_0]], align 8
+; CHECK-NEXT: [[LD0_0:%.*]] = load double, ptr [[ARG1:%.*]], align 8
+; CHECK-NEXT: [[MUL1_0:%.*]] = fmul fast double [[LD0_0]], [[LD1_0]]
+; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 16
+; CHECK-NEXT: [[LD2_0:%.*]] = load double, ptr [[GEP2_0]], align 8
+; CHECK-NEXT: [[MUL2_0:%.*]] = fmul fast double [[LD2_0]], [[LD1_0]]
+; CHECK-NEXT: [[GEP1_1:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 3
+; CHECK-NEXT: [[LD1_1:%.*]] = load double, ptr [[GEP1_1]], align 8
+; CHECK-NEXT: [[GEP0_1:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 1
+; CHECK-NEXT: [[LD0_1:%.*]] = load double, ptr [[GEP0_1]], align 8
+; CHECK-NEXT: [[MUL1_1:%.*]] = fmul fast double [[LD0_1]], [[LD1_1]]
+; CHECK-NEXT: [[RDX1_0:%.*]] = fadd fast double [[MUL1_0]], [[MUL1_1]]
+; CHECK-NEXT: [[GEP2_1:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 17
+; CHECK-NEXT: [[LD2_1:%.*]] = load double, ptr [[GEP2_1]], align 8
+; CHECK-NEXT: [[MUL2_1:%.*]] = fmul fast double [[LD2_1]], [[LD1_1]]
+; CHECK-NEXT: [[RDX2_0:%.*]] = fadd fast double [[MUL2_0]], [[MUL2_1]]
+; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 5
+; CHECK-NEXT: [[LD1_2:%.*]] = load double, ptr [[GEP1_2]], align 8
+; CHECK-NEXT: [[GEP0_2:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 2
+; CHECK-NEXT: [[LD0_2:%.*]] = load double, ptr [[GEP0_2]], align 8
+; CHECK-NEXT: [[MUL1_2:%.*]] = fmul fast double [[LD0_2]], [[LD1_2]]
+; CHECK-NEXT: [[RDX1_1:%.*]] = fadd fast double [[RDX1_0]], [[MUL1_2]]
+; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 18
+; CHECK-NEXT: [[LD2_2:%.*]] = load double, ptr [[GEP2_2]], align 8
+; CHECK-NEXT: [[MUL2_2:%.*]] = fmul fast double [[LD2_2]], [[LD1_2]]
+; CHECK-NEXT: [[RDX2_1:%.*]] = fadd fast double [[RDX2_0]], [[MUL2_2]]
+; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 7
+; CHECK-NEXT: [[LD1_3:%.*]] = load double, ptr [[GEP1_3]], align 8
+; CHECK-NEXT: [[GEP0_3:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 3
+; CHECK-NEXT: [[LD0_3:%.*]] = load double, ptr [[GEP0_3]], align 8
+; CHECK-NEXT: [[MUL1_3:%.*]] = fmul fast double [[LD0_3]], [[LD1_3]]
+; CHECK-NEXT: [[RDX1_2:%.*]] = fadd fast double [[RDX1_1]], [[MUL1_3]]
+; CHECK-NEXT: [[GEP2_3:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 19
+; CHECK-NEXT: [[LD2_3:%.*]] = load double, ptr [[GEP2_3]], align 8
+; CHECK-NEXT: [[MUL2_3:%.*]] = fmul fast double [[LD2_3]], [[LD1_3]]
+; CHECK-NEXT: [[RDX2_2:%.*]] = fadd fast double [[RDX2_1]], [[MUL2_3]]
+; CHECK-NEXT: [[GEP1_4:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 9
+; CHECK-NEXT: [[LD1_4:%.*]] = load double, ptr [[GEP1_4]], align 8
+; CHECK-NEXT: [[GEP0_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 4
+; CHECK-NEXT: [[LD0_4:%.*]] = load double, ptr [[GEP0_4]], align 8
+; CHECK-NEXT: [[MUL1_4:%.*]] = fmul fast double [[LD0_4]], [[LD1_4]]
+; CHECK-NEXT: [[RDX1_3:%.*]] = fadd fast double [[RDX1_2]], [[MUL1_4]]
+; CHECK-NEXT: [[GEP2_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 20
+; CHECK-NEXT: [[LD2_4:%.*]] = load double, ptr [[GEP2_4]], align 8
+; CHECK-NEXT: [[MUL2_4:%.*]] = fmul fast double [[LD2_4]], [[LD1_4]]
+; CHECK-NEXT: [[RDX2_3:%.*]] = fadd fast double [[RDX2_2]], [[MUL2_4]]
+; CHECK-NEXT: [[GEP1_5:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 11
+; CHECK-NEXT: [[LD1_5:%.*]] = load double, ptr [[GEP1_5]], align 8
+; CHECK-NEXT: [[GEP0_5:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 5
+; CHECK-NEXT: [[LD0_5:%.*]] = load double, ptr [[GEP0_5]], align 8
+; CHECK-NEXT: [[MUL1_5:%.*]] = fmul fast double [[LD0_5]], [[LD1_5]]
+; CHECK-NEXT: [[RDX1_4:%.*]] = fadd fast double [[RDX1_3]], [[MUL1_5]]
+; CHECK-NEXT: [[GEP2_5:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 21
+; CHECK-NEXT: [[LD2_5:%.*]] = load double, ptr [[GEP2_5]], align 8
+; CHECK-NEXT: [[MUL2_5:%.*]] = fmul fast double [[LD2_5]], [[LD1_5]]
+; CHECK-NEXT: [[RDX2_4:%.*]] = fadd fast double [[RDX2_3]], [[MUL2_5]]
+; CHECK-NEXT: [[GEP1_6:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 13
+; CHECK-NEXT: [[LD1_6:%.*]] = load double, ptr [[GEP1_6]], align 8
+; CHECK-NEXT: [[GEP0_6:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 6
+; CHECK-NEXT: [[LD0_6:%.*]] = load double, ptr [[GEP0_6]], align 8
+; CHECK-NEXT: [[MUL1_6:%.*]] = fmul fast double [[LD0_6]], [[LD1_6]]
+; CHECK-NEXT: [[RDX1_5:%.*]] = fadd fast double [[RDX1_4]], [[MUL1_6]]
+; CHECK-NEXT: [[GEP2_6:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 22
+; CHECK-NEXT: [[LD2_6:%.*]] = load double, ptr [[GEP2_6]], align 8
+; CHECK-NEXT: [[MUL2_6:%.*]] = fmul fast double [[LD2_6]], [[LD1_6]]
+; CHECK-NEXT: [[RDX2_5:%.*]] = fadd fast double [[RDX2_4]], [[MUL2_6]]
+; CHECK-NEXT: [[GEP1_7:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 15
+; CHECK-NEXT: [[LD1_7:%.*]] = load double, ptr [[GEP1_7]], align 8
+; CHECK-NEXT: [[GEP0_7:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 7
+; CHECK-NEXT: [[LD0_7:%.*]] = load double, ptr [[GEP0_7]], align 8
+; CHECK-NEXT: [[MUL1_7:%.*]] = fmul fast double [[LD0_7]], [[LD1_7]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double [[RDX1_5]], [[MUL1_7]]
+; CHECK-NEXT: [[GEP2_7:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 23
+; CHECK-NEXT: [[LD2_7:%.*]] = load double, ptr [[GEP2_7]], align 8
+; CHECK-NEXT: [[MUL2_7:%.*]] = fmul fast double [[LD2_7]], [[LD1_7]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd fast double [[RDX2_5]], [[MUL2_7]]
; CHECK-NEXT: [[I142:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
; CHECK-NEXT: [[I143:%.*]] = insertelement <2 x double> [[I142]], double [[TMP11]], i64 1
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds double, ptr [[ARG2:%.*]], <2 x i64> <i64 0, i64 16>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
index 979701609b499..bb10829b8cca2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
@@ -12,30 +12,33 @@ define void @select_logical_or_i1(ptr %dst,
; CHECK-LABEL: define void @select_logical_or_i1(
; CHECK-SAME: ptr [[DST:%.*]], float [[D0:%.*]], float [[D1:%.*]], float [[D2:%.*]], float [[D3:%.*]], float [[THRESHOLD:%.*]], float [[HPHB_VAL:%.*]], i1 [[SCALAR_COND:%.*]], float [[Y0:%.*]], float [[Y1:%.*]], float [[Y2:%.*]], float [[Y3:%.*]], float [[E0:%.*]], float [[E1:%.*]], float [[E2:%.*]], float [[E3:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[D0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[D1]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[D2]], i64 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D3]], i64 3
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp fast uge <4 x float> [[TMP3]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> poison, i1 [[SCALAR_COND]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> poison, float [[HPHB_VAL]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x float> zeroinitializer, <4 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[Y0]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
-; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
-; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
-; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
+; CHECK-NEXT: [[CMP0:%.*]] = fcmp fast uge float [[D0]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast uge float [[D1]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast uge float [[D2]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast uge float [[D3]], [[THRESHOLD]]
+; CHECK-NEXT: [[OR0:%.*]] = select i1 [[CMP0]], i1 true, i1 [[SCALAR_COND]]
+; CHECK-NEXT: [[OR1:%.*]] = select i1 [[CMP1]], i1 true, i1 [[SCALAR_COND]]
+; CHECK-NEXT: [[OR2:%.*]] = select i1 [[CMP2]], i1 true, i1 [[SCALAR_COND]]
+; CHECK-NEXT: [[OR3:%.*]] = select i1 [[CMP3]], i1 true, i1 [[SCALAR_COND]]
+; CHECK-NEXT: [[SEL0:%.*]] = select i1 [[OR0]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[OR1]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[OR2]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL3:%.*]] = select i1 [[OR3]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[MUL0:%.*]] = fmul fast float [[SEL0]], [[Y0]]
+; CHECK-NEXT: [[MUL1:%.*]] = fmul fast float [[SEL1]], [[Y1]]
+; CHECK-NEXT: [[MUL2:%.*]] = fmul fast float [[SEL2]], [[Y2]]
+; CHECK-NEXT: [[MUL3:%.*]] = fmul fast float [[SEL3]], [[Y3]]
+; CHECK-NEXT: [[RES0:%.*]] = fadd fast float [[E0]], [[MUL0]]
+; CHECK-NEXT: [[RES1:%.*]] = fadd fast float [[E1]], [[MUL1]]
+; CHECK-NEXT: [[RES2:%.*]] = fadd fast float [[E2]], [[MUL2]]
+; CHECK-NEXT: [[RES3:%.*]] = fadd fast float [[E3]], [[MUL3]]
+; CHECK-NEXT: store float [[RES0]], ptr [[DST]], align 4
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 1
+; CHECK-NEXT: store float [[RES1]], ptr [[P1]], align 4
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
+; CHECK-NEXT: store float [[RES2]], ptr [[P2]], align 4
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
+; CHECK-NEXT: store float [[RES3]], ptr [[P3]], align 4
; CHECK-NEXT: ret void
;
float %d0, float %d1, float %d2, float %d3,
@@ -84,30 +87,33 @@ define void @select_logical_and_i1(ptr %dst,
; CHECK-LABEL: define void @select_logical_and_i1(
; CHECK-SAME: ptr [[DST:%.*]], float [[D0:%.*]], float [[D1:%.*]], float [[D2:%.*]], float [[D3:%.*]], float [[THRESHOLD:%.*]], float [[HPHB_VAL:%.*]], i1 [[SCALAR_COND:%.*]], float [[Y0:%.*]], float [[Y1:%.*]], float [[Y2:%.*]], float [[Y3:%.*]], float [[E0:%.*]], float [[E1:%.*]], float [[E2:%.*]], float [[E3:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[D0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[D1]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[D2]], i64 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D3]], i64 3
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp fast uge <4 x float> [[TMP3]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> poison, i1 [[SCALAR_COND]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> poison, float [[HPHB_VAL]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x float> zeroinitializer, <4 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[Y0]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
-; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
-; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
-; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
+; CHECK-NEXT: [[CMP0:%.*]] = fcmp fast uge float [[D0]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast uge float [[D1]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast uge float [[D2]], [[THRESHOLD]]
+; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast uge float [[D3]], [[THRESHOLD]]
+; CHECK-NEXT: [[AND0:%.*]] = select i1 [[CMP0]], i1 [[SCALAR_COND]], i1 false
+; CHECK-NEXT: [[AND1:%.*]] = select i1 [[CMP1]], i1 [[SCALAR_COND]], i1 false
+; CHECK-NEXT: [[AND2:%.*]] = select i1 [[CMP2]], i1 [[SCALAR_COND]], i1 false
+; CHECK-NEXT: [[AND3:%.*]] = select i1 [[CMP3]], i1 [[SCALAR_COND]], i1 false
+; CHECK-NEXT: [[SEL0:%.*]] = select i1 [[AND0]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[AND1]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[AND2]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[SEL3:%.*]] = select i1 [[AND3]], float 0.000000e+00, float [[HPHB_VAL]]
+; CHECK-NEXT: [[MUL0:%.*]] = fmul fast float [[SEL0]], [[Y0]]
+; CHECK-NEXT: [[MUL1:%.*]] = fmul fast float [[SEL1]], [[Y1]]
+; CHECK-NEXT: [[MUL2:%.*]] = fmul fast float [[SEL2]], [[Y2]]
+; CHECK-NEXT: [[MUL3:%.*]] = fmul fast float [[SEL3]], [[Y3]]
+; CHECK-NEXT: [[RES0:%.*]] = fadd fast float [[E0]], [[MUL0]]
+; CHECK-NEXT: [[RES1:%.*]] = fadd fast float [[E1]], [[MUL1]]
+; CHECK-NEXT: [[RES2:%.*]] = fadd fast float [[E2]], [[MUL2]]
+; CHECK-NEXT: [[RES3:%.*]] = fadd fast float [[E3]], [[MUL3]]
+; CHECK-NEXT: store float [[RES0]], ptr [[DST]], align 4
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 1
+; CHECK-NEXT: store float [[RES1]], ptr [[P1]], align 4
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
+; CHECK-NEXT: store float [[RES2]], ptr [[P2]], align 4
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
+; CHECK-NEXT: store float [[RES3]], ptr [[P3]], align 4
; CHECK-NEXT: ret void
;
float %d0, float %d1, float %d2, float %d3,
>From 5acbc72a73c39550634514a5cc0b7874ad3c85b6 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 17 Aug 2026 23:58:49 +0200
Subject: [PATCH 09/13] Revert "showcase for revert AllowReassoc"
This reverts commit 796c8dddef5b7bf1a4e49b3d732addb1fabf4af4.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 9 +-
.../AMDGPU/elementwise-fma-operand1.ll | 168 ++++++++++++------
.../X86/horizontal-fadd-with-sub.ll | 34 ++--
.../X86/redux-feed-buildvector.ll | 88 ++-------
.../X86/select-logical-or-and-i1-vector.ll | 102 +++++------
5 files changed, 193 insertions(+), 208 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4c0784e20c48c..28699b2664387 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14425,15 +14425,18 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
// The fmul may sit on either side of the add/sub. Look past operand 0 only
- // for chains that allow reassociation. An fsub can only fold an fmul on its
- // left, so stop at operand 0 there as well.
+ // for chains that do not allow reassociation. The gate is profitability, not
+ // correctness. A reassociative chain can be vectorized into a vector fmul
+ // feeding a reduction, which is usually better than the scalar fma chain
+ // this check protects. An fsub can only fold an fmul on its left, so stop at
+ // operand 0 there as well.
bool AllowReassoc = all_of(VL, [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
if (!I || S.isCopyableElement(I))
return true;
return match(I, m_AllowReassoc(m_Value()));
});
- bool OnlyFirstOperand = !AllowReassoc || S.getOpcode() == Instruction::FSub;
+ bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
unsigned NumCandidateOps =
OnlyFirstOperand ? 1
: getNumberOfPotentiallyCommutativeOps(S.getMainOp());
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
index daee269eef852..eed8216c5b1d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -18,12 +18,42 @@ define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr
; CHECK-LABEL: define void @axpy4_contract(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
-; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[R0:%.*]] = fadd contract float [[C0]], [[M0]]
+; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
+; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
+; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_contract(
@@ -81,42 +111,12 @@ define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr n
; CHECK-LABEL: define void @axpy4_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT: [[M0:%.*]] = fmul reassoc contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
-; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
-; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul reassoc contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[R1:%.*]] = fadd reassoc contract float [[C1]], [[M1]]
-; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; CHECK-NEXT: [[M2:%.*]] = fmul reassoc contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[R2:%.*]] = fadd reassoc contract float [[C2]], [[M2]]
-; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; CHECK-NEXT: [[M3:%.*]] = fmul reassoc contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[R3:%.*]] = fadd reassoc contract float [[C3]], [[M3]]
-; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc contract <4 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_reassoc(
@@ -174,23 +174,83 @@ define void @axpy4_mixed_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b,
; CHECK-LABEL: define void @axpy4_mixed_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
-; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
+; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
+; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
+; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
+; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_mixed_reassoc(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR12-NEXT: [[ENTRY:.*:]]
-; THR12-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
-; THR12-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
-; THR12-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
-; THR12-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; THR12-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
-; THR12-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
+; THR12-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
+; THR12-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; THR12-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
+; THR12-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; THR12-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
+; THR12-NEXT: store float [[R0]], ptr [[D]], align 4
+; THR12-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; THR12-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; THR12-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; THR12-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; THR12-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; THR12-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; THR12-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
+; THR12-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
+; THR12-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; THR12-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; THR12-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; THR12-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; THR12-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; THR12-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; THR12-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; THR12-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; THR12-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
+; THR12-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
+; THR12-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; THR12-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; THR12-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; THR12-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; THR12-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; THR12-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; THR12-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; THR12-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; THR12-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
+; THR12-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
+; THR12-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; THR12-NEXT: store float [[R3]], ptr [[DP3]], align 4
; THR12-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
index 96dbc39a5b780..b1b163e64dc79 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
@@ -9,19 +9,16 @@ define double @fsub_fmul_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_fmul_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fmul reassoc nsz contract double [[Y1]], [[X1]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
; CHECK-NEXT: [[ZSUM:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc nsz contract double [[TMP5]], [[ZSUM]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc nsz contract double [[SUB]], [[TMP4]]
; CHECK-NEXT: ret double [[TMP6]]
;
@@ -49,28 +46,25 @@ define double @fsub_fmul_4(ptr %x, ptr %y, ptr %z) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[X16:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[Y16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 16
; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 24
; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 24
; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y8]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
+; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
; CHECK-NEXT: [[MUL16:%.*]] = fmul reassoc nsz contract double [[Y3]], [[X3]]
-; CHECK-NEXT: [[X2:%.*]] = load double, ptr [[X16]], align 8
-; CHECK-NEXT: [[Y2:%.*]] = load double, ptr [[Y16]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fmul reassoc nsz contract double [[Y2]], [[X2]]
-; CHECK-NEXT: [[X4:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y4:%.*]] = load double, ptr [[Y24]], align 8
-; CHECK-NEXT: [[MUL17:%.*]] = fmul reassoc nsz contract double [[Y4]], [[X4]]
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[Z]], align 8
-; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[MUL]], [[MUL16]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[MUL]], [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[T3:%.*]] = fadd reassoc nsz contract double [[T1]], [[TMP7]]
; CHECK-NEXT: [[TMP6:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
; CHECK-NEXT: [[ADD13:%.*]] = fsub reassoc nsz contract double [[T3]], [[TMP6]]
-; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[ADD13]], [[MUL17]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[ADD13]], [[MUL16]]
; CHECK-NEXT: ret double [[TMP5]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
index 1801c14a72982..67054d202d5a2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
@@ -10,83 +10,17 @@ declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32 immarg
define void @test(ptr nocapture readonly %arg, ptr nocapture readonly %arg1, ptr nocapture %arg2) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_0:%.*]] = getelementptr inbounds double, ptr [[ARG:%.*]], i64 1
-; CHECK-NEXT: [[LD1_0:%.*]] = load double, ptr [[GEP1_0]], align 8
-; CHECK-NEXT: [[LD0_0:%.*]] = load double, ptr [[ARG1:%.*]], align 8
-; CHECK-NEXT: [[MUL1_0:%.*]] = fmul fast double [[LD0_0]], [[LD1_0]]
-; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 16
-; CHECK-NEXT: [[LD2_0:%.*]] = load double, ptr [[GEP2_0]], align 8
-; CHECK-NEXT: [[MUL2_0:%.*]] = fmul fast double [[LD2_0]], [[LD1_0]]
-; CHECK-NEXT: [[GEP1_1:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 3
-; CHECK-NEXT: [[LD1_1:%.*]] = load double, ptr [[GEP1_1]], align 8
-; CHECK-NEXT: [[GEP0_1:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 1
-; CHECK-NEXT: [[LD0_1:%.*]] = load double, ptr [[GEP0_1]], align 8
-; CHECK-NEXT: [[MUL1_1:%.*]] = fmul fast double [[LD0_1]], [[LD1_1]]
-; CHECK-NEXT: [[RDX1_0:%.*]] = fadd fast double [[MUL1_0]], [[MUL1_1]]
-; CHECK-NEXT: [[GEP2_1:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 17
-; CHECK-NEXT: [[LD2_1:%.*]] = load double, ptr [[GEP2_1]], align 8
-; CHECK-NEXT: [[MUL2_1:%.*]] = fmul fast double [[LD2_1]], [[LD1_1]]
-; CHECK-NEXT: [[RDX2_0:%.*]] = fadd fast double [[MUL2_0]], [[MUL2_1]]
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 5
-; CHECK-NEXT: [[LD1_2:%.*]] = load double, ptr [[GEP1_2]], align 8
-; CHECK-NEXT: [[GEP0_2:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 2
-; CHECK-NEXT: [[LD0_2:%.*]] = load double, ptr [[GEP0_2]], align 8
-; CHECK-NEXT: [[MUL1_2:%.*]] = fmul fast double [[LD0_2]], [[LD1_2]]
-; CHECK-NEXT: [[RDX1_1:%.*]] = fadd fast double [[RDX1_0]], [[MUL1_2]]
-; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 18
-; CHECK-NEXT: [[LD2_2:%.*]] = load double, ptr [[GEP2_2]], align 8
-; CHECK-NEXT: [[MUL2_2:%.*]] = fmul fast double [[LD2_2]], [[LD1_2]]
-; CHECK-NEXT: [[RDX2_1:%.*]] = fadd fast double [[RDX2_0]], [[MUL2_2]]
-; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 7
-; CHECK-NEXT: [[LD1_3:%.*]] = load double, ptr [[GEP1_3]], align 8
-; CHECK-NEXT: [[GEP0_3:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 3
-; CHECK-NEXT: [[LD0_3:%.*]] = load double, ptr [[GEP0_3]], align 8
-; CHECK-NEXT: [[MUL1_3:%.*]] = fmul fast double [[LD0_3]], [[LD1_3]]
-; CHECK-NEXT: [[RDX1_2:%.*]] = fadd fast double [[RDX1_1]], [[MUL1_3]]
-; CHECK-NEXT: [[GEP2_3:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 19
-; CHECK-NEXT: [[LD2_3:%.*]] = load double, ptr [[GEP2_3]], align 8
-; CHECK-NEXT: [[MUL2_3:%.*]] = fmul fast double [[LD2_3]], [[LD1_3]]
-; CHECK-NEXT: [[RDX2_2:%.*]] = fadd fast double [[RDX2_1]], [[MUL2_3]]
-; CHECK-NEXT: [[GEP1_4:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 9
-; CHECK-NEXT: [[LD1_4:%.*]] = load double, ptr [[GEP1_4]], align 8
-; CHECK-NEXT: [[GEP0_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 4
-; CHECK-NEXT: [[LD0_4:%.*]] = load double, ptr [[GEP0_4]], align 8
-; CHECK-NEXT: [[MUL1_4:%.*]] = fmul fast double [[LD0_4]], [[LD1_4]]
-; CHECK-NEXT: [[RDX1_3:%.*]] = fadd fast double [[RDX1_2]], [[MUL1_4]]
-; CHECK-NEXT: [[GEP2_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 20
-; CHECK-NEXT: [[LD2_4:%.*]] = load double, ptr [[GEP2_4]], align 8
-; CHECK-NEXT: [[MUL2_4:%.*]] = fmul fast double [[LD2_4]], [[LD1_4]]
-; CHECK-NEXT: [[RDX2_3:%.*]] = fadd fast double [[RDX2_2]], [[MUL2_4]]
-; CHECK-NEXT: [[GEP1_5:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 11
-; CHECK-NEXT: [[LD1_5:%.*]] = load double, ptr [[GEP1_5]], align 8
-; CHECK-NEXT: [[GEP0_5:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 5
-; CHECK-NEXT: [[LD0_5:%.*]] = load double, ptr [[GEP0_5]], align 8
-; CHECK-NEXT: [[MUL1_5:%.*]] = fmul fast double [[LD0_5]], [[LD1_5]]
-; CHECK-NEXT: [[RDX1_4:%.*]] = fadd fast double [[RDX1_3]], [[MUL1_5]]
-; CHECK-NEXT: [[GEP2_5:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 21
-; CHECK-NEXT: [[LD2_5:%.*]] = load double, ptr [[GEP2_5]], align 8
-; CHECK-NEXT: [[MUL2_5:%.*]] = fmul fast double [[LD2_5]], [[LD1_5]]
-; CHECK-NEXT: [[RDX2_4:%.*]] = fadd fast double [[RDX2_3]], [[MUL2_5]]
-; CHECK-NEXT: [[GEP1_6:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 13
-; CHECK-NEXT: [[LD1_6:%.*]] = load double, ptr [[GEP1_6]], align 8
-; CHECK-NEXT: [[GEP0_6:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 6
-; CHECK-NEXT: [[LD0_6:%.*]] = load double, ptr [[GEP0_6]], align 8
-; CHECK-NEXT: [[MUL1_6:%.*]] = fmul fast double [[LD0_6]], [[LD1_6]]
-; CHECK-NEXT: [[RDX1_5:%.*]] = fadd fast double [[RDX1_4]], [[MUL1_6]]
-; CHECK-NEXT: [[GEP2_6:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 22
-; CHECK-NEXT: [[LD2_6:%.*]] = load double, ptr [[GEP2_6]], align 8
-; CHECK-NEXT: [[MUL2_6:%.*]] = fmul fast double [[LD2_6]], [[LD1_6]]
-; CHECK-NEXT: [[RDX2_5:%.*]] = fadd fast double [[RDX2_4]], [[MUL2_6]]
-; CHECK-NEXT: [[GEP1_7:%.*]] = getelementptr inbounds double, ptr [[ARG]], i64 15
-; CHECK-NEXT: [[LD1_7:%.*]] = load double, ptr [[GEP1_7]], align 8
-; CHECK-NEXT: [[GEP0_7:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 7
-; CHECK-NEXT: [[LD0_7:%.*]] = load double, ptr [[GEP0_7]], align 8
-; CHECK-NEXT: [[MUL1_7:%.*]] = fmul fast double [[LD0_7]], [[LD1_7]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double [[RDX1_5]], [[MUL1_7]]
-; CHECK-NEXT: [[GEP2_7:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 23
-; CHECK-NEXT: [[LD2_7:%.*]] = load double, ptr [[GEP2_7]], align 8
-; CHECK-NEXT: [[MUL2_7:%.*]] = fmul fast double [[LD2_7]], [[LD1_7]]
-; CHECK-NEXT: [[TMP11:%.*]] = fadd fast double [[RDX2_5]], [[MUL2_7]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x ptr> poison, ptr [[ARG:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x ptr> [[TMP0]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP1]], <8 x i64> <i64 1, i64 3, i64 5, i64 7, i64 9, i64 11, i64 13, i64 15>
+; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1:%.*]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP5]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[GEP2_0]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <8 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x double> [[TMP6]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
+; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
; CHECK-NEXT: [[I142:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
; CHECK-NEXT: [[I143:%.*]] = insertelement <2 x double> [[I142]], double [[TMP11]], i64 1
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds double, ptr [[ARG2:%.*]], <2 x i64> <i64 0, i64 16>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
index bb10829b8cca2..979701609b499 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
@@ -12,33 +12,30 @@ define void @select_logical_or_i1(ptr %dst,
; CHECK-LABEL: define void @select_logical_or_i1(
; CHECK-SAME: ptr [[DST:%.*]], float [[D0:%.*]], float [[D1:%.*]], float [[D2:%.*]], float [[D3:%.*]], float [[THRESHOLD:%.*]], float [[HPHB_VAL:%.*]], i1 [[SCALAR_COND:%.*]], float [[Y0:%.*]], float [[Y1:%.*]], float [[Y2:%.*]], float [[Y3:%.*]], float [[E0:%.*]], float [[E1:%.*]], float [[E2:%.*]], float [[E3:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[CMP0:%.*]] = fcmp fast uge float [[D0]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast uge float [[D1]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast uge float [[D2]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast uge float [[D3]], [[THRESHOLD]]
-; CHECK-NEXT: [[OR0:%.*]] = select i1 [[CMP0]], i1 true, i1 [[SCALAR_COND]]
-; CHECK-NEXT: [[OR1:%.*]] = select i1 [[CMP1]], i1 true, i1 [[SCALAR_COND]]
-; CHECK-NEXT: [[OR2:%.*]] = select i1 [[CMP2]], i1 true, i1 [[SCALAR_COND]]
-; CHECK-NEXT: [[OR3:%.*]] = select i1 [[CMP3]], i1 true, i1 [[SCALAR_COND]]
-; CHECK-NEXT: [[SEL0:%.*]] = select i1 [[OR0]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[OR1]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[OR2]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL3:%.*]] = select i1 [[OR3]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[MUL0:%.*]] = fmul fast float [[SEL0]], [[Y0]]
-; CHECK-NEXT: [[MUL1:%.*]] = fmul fast float [[SEL1]], [[Y1]]
-; CHECK-NEXT: [[MUL2:%.*]] = fmul fast float [[SEL2]], [[Y2]]
-; CHECK-NEXT: [[MUL3:%.*]] = fmul fast float [[SEL3]], [[Y3]]
-; CHECK-NEXT: [[RES0:%.*]] = fadd fast float [[E0]], [[MUL0]]
-; CHECK-NEXT: [[RES1:%.*]] = fadd fast float [[E1]], [[MUL1]]
-; CHECK-NEXT: [[RES2:%.*]] = fadd fast float [[E2]], [[MUL2]]
-; CHECK-NEXT: [[RES3:%.*]] = fadd fast float [[E3]], [[MUL3]]
-; CHECK-NEXT: store float [[RES0]], ptr [[DST]], align 4
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 1
-; CHECK-NEXT: store float [[RES1]], ptr [[P1]], align 4
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
-; CHECK-NEXT: store float [[RES2]], ptr [[P2]], align 4
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT: store float [[RES3]], ptr [[P3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[D0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[D1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[D2]], i64 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D3]], i64 3
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp fast uge <4 x float> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> poison, i1 [[SCALAR_COND]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> poison, float [[HPHB_VAL]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x float> zeroinitializer, <4 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[Y0]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
+; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
+; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
+; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
float %d0, float %d1, float %d2, float %d3,
@@ -87,33 +84,30 @@ define void @select_logical_and_i1(ptr %dst,
; CHECK-LABEL: define void @select_logical_and_i1(
; CHECK-SAME: ptr [[DST:%.*]], float [[D0:%.*]], float [[D1:%.*]], float [[D2:%.*]], float [[D3:%.*]], float [[THRESHOLD:%.*]], float [[HPHB_VAL:%.*]], i1 [[SCALAR_COND:%.*]], float [[Y0:%.*]], float [[Y1:%.*]], float [[Y2:%.*]], float [[Y3:%.*]], float [[E0:%.*]], float [[E1:%.*]], float [[E2:%.*]], float [[E3:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[CMP0:%.*]] = fcmp fast uge float [[D0]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast uge float [[D1]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP2:%.*]] = fcmp fast uge float [[D2]], [[THRESHOLD]]
-; CHECK-NEXT: [[CMP3:%.*]] = fcmp fast uge float [[D3]], [[THRESHOLD]]
-; CHECK-NEXT: [[AND0:%.*]] = select i1 [[CMP0]], i1 [[SCALAR_COND]], i1 false
-; CHECK-NEXT: [[AND1:%.*]] = select i1 [[CMP1]], i1 [[SCALAR_COND]], i1 false
-; CHECK-NEXT: [[AND2:%.*]] = select i1 [[CMP2]], i1 [[SCALAR_COND]], i1 false
-; CHECK-NEXT: [[AND3:%.*]] = select i1 [[CMP3]], i1 [[SCALAR_COND]], i1 false
-; CHECK-NEXT: [[SEL0:%.*]] = select i1 [[AND0]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[AND1]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[AND2]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[SEL3:%.*]] = select i1 [[AND3]], float 0.000000e+00, float [[HPHB_VAL]]
-; CHECK-NEXT: [[MUL0:%.*]] = fmul fast float [[SEL0]], [[Y0]]
-; CHECK-NEXT: [[MUL1:%.*]] = fmul fast float [[SEL1]], [[Y1]]
-; CHECK-NEXT: [[MUL2:%.*]] = fmul fast float [[SEL2]], [[Y2]]
-; CHECK-NEXT: [[MUL3:%.*]] = fmul fast float [[SEL3]], [[Y3]]
-; CHECK-NEXT: [[RES0:%.*]] = fadd fast float [[E0]], [[MUL0]]
-; CHECK-NEXT: [[RES1:%.*]] = fadd fast float [[E1]], [[MUL1]]
-; CHECK-NEXT: [[RES2:%.*]] = fadd fast float [[E2]], [[MUL2]]
-; CHECK-NEXT: [[RES3:%.*]] = fadd fast float [[E3]], [[MUL3]]
-; CHECK-NEXT: store float [[RES0]], ptr [[DST]], align 4
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 1
-; CHECK-NEXT: store float [[RES1]], ptr [[P1]], align 4
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
-; CHECK-NEXT: store float [[RES2]], ptr [[P2]], align 4
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT: store float [[RES3]], ptr [[P3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[D0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[D1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[D2]], i64 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D3]], i64 3
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp fast uge <4 x float> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> poison, i1 [[SCALAR_COND]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> poison, float [[HPHB_VAL]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x float> zeroinitializer, <4 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[Y0]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
+; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
+; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
+; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
float %d0, float %d1, float %d2, float %d3,
>From c53eba5f2c54ffbae131695207d61b6c43cb237c Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 18 Aug 2026 21:15:12 +0200
Subject: [PATCH 10/13] Drop the operand-1 search from this patch
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 43 ++----
.../AMDGPU/elementwise-fma-operand1.ll | 126 +++---------------
2 files changed, 27 insertions(+), 142 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 28699b2664387..320a2c611eeeb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14424,38 +14424,14 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
- // The fmul may sit on either side of the add/sub. Look past operand 0 only
- // for chains that do not allow reassociation. The gate is profitability, not
- // correctness. A reassociative chain can be vectorized into a vector fmul
- // feeding a reduction, which is usually better than the scalar fma chain
- // this check protects. An fsub can only fold an fmul on its left, so stop at
- // operand 0 there as well.
- bool AllowReassoc = all_of(VL, [&](Value *V) {
- auto *I = dyn_cast<Instruction>(V);
- if (!I || S.isCopyableElement(I))
- return true;
- return match(I, m_AllowReassoc(m_Value()));
- });
- bool OnlyFirstOperand = AllowReassoc || S.getOpcode() == Instruction::FSub;
- unsigned NumCandidateOps =
- OnlyFirstOperand ? 1
- : getNumberOfPotentiallyCommutativeOps(S.getMainOp());
- auto GetFMulOperandIdx = [&]() -> std::optional<unsigned> {
- for (unsigned Idx : seq<unsigned>(0, NumCandidateOps)) {
- InstructionsState CandS = getSameOpcode(Operands[Idx], TLI);
- if (!CandS.valid() || CandS.isAltShuffle() ||
- CandS.getOpcode() != Instruction::FMul)
- continue;
- if (!CheckForContractable(Operands[Idx], CandS))
- continue;
- return Idx;
- }
- return std::nullopt;
- };
- std::optional<unsigned> FMulIdx = GetFMulOperandIdx();
- if (!FMulIdx)
+ InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
+ if (!OpS.valid())
+ return InstructionCost::getInvalid();
+
+ if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
+ return InstructionCost::getInvalid();
+ if (!CheckForContractable(Operands.front(), OpS))
return InstructionCost::getInvalid();
- InstructionsState OpS = getSameOpcode(Operands[*FMulIdx], TLI);
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
@@ -14484,7 +14460,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
TTI.getArithmeticInstrCost(S.getOpcode(), I->getType(), CostKind);
}
unsigned NumOps = 0;
- for (auto [V, Op] : zip(VL, Operands[*FMulIdx])) {
+ for (auto [V, Op] : zip(VL, Operands.front())) {
if (S.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(Op);
@@ -15303,8 +15279,7 @@ void BoUpSLP::transformNodes() {
break;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
- TreeEntry *FMulEntry =
- getOperandEntry(&E, IsOneUseVectorFMulOperand(LHS) ? 0 : 1);
+ TreeEntry *FMulEntry = getOperandEntry(&E, 0);
if (FMulEntry->UserTreeIndex &&
FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
index eed8216c5b1d2..c4c97622ac726 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll
@@ -18,42 +18,12 @@ define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr
; CHECK-LABEL: define void @axpy4_contract(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[R0:%.*]] = fadd contract float [[C0]], [[M0]]
-; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
-; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_contract(
@@ -174,83 +144,23 @@ define void @axpy4_mixed_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b,
; CHECK-LABEL: define void @axpy4_mixed_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; CHECK-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
-; CHECK-NEXT: store float [[R0]], ptr [[D]], align 4
-; CHECK-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; CHECK-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; CHECK-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; CHECK-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; CHECK-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; CHECK-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; CHECK-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; CHECK-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; CHECK-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; CHECK-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; CHECK-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; CHECK-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; CHECK-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; CHECK-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT: ret void
;
; THR12-LABEL: define void @axpy4_mixed_reassoc(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR12-NEXT: [[ENTRY:.*:]]
-; THR12-NEXT: [[C0:%.*]] = load float, ptr [[C]], align 4
-; THR12-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
-; THR12-NEXT: [[B0:%.*]] = load float, ptr [[B]], align 4
-; THR12-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
-; THR12-NEXT: [[R0:%.*]] = fadd reassoc contract float [[C0]], [[M0]]
-; THR12-NEXT: store float [[R0]], ptr [[D]], align 4
-; THR12-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
-; THR12-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
-; THR12-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; THR12-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
-; THR12-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
-; THR12-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
-; THR12-NEXT: [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
-; THR12-NEXT: [[R1:%.*]] = fadd contract float [[C1]], [[M1]]
-; THR12-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
-; THR12-NEXT: store float [[R1]], ptr [[DP1]], align 4
-; THR12-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
-; THR12-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
-; THR12-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; THR12-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
-; THR12-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; THR12-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
-; THR12-NEXT: [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
-; THR12-NEXT: [[R2:%.*]] = fadd contract float [[C2]], [[M2]]
-; THR12-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
-; THR12-NEXT: store float [[R2]], ptr [[DP2]], align 4
-; THR12-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
-; THR12-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
-; THR12-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
-; THR12-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
-; THR12-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
-; THR12-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
-; THR12-NEXT: [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
-; THR12-NEXT: [[R3:%.*]] = fadd contract float [[C3]], [[M3]]
-; THR12-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
-; THR12-NEXT: store float [[R3]], ptr [[DP3]], align 4
+; THR12-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
+; THR12-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
+; THR12-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
+; THR12-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; THR12-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
+; THR12-NEXT: store <4 x float> [[TMP4]], ptr [[D]], align 4
; THR12-NEXT: ret void
;
entry:
>From f56844f7ef796b602c87c3a99bc607187d89fc3d Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 21 Aug 2026 01:27:15 +0200
Subject: [PATCH 11/13] address comments
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a35b13805218f..080da069d527d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14506,11 +14506,19 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
auto *I = dyn_cast<Instruction>(V);
if (!I)
continue;
- if (!S.isCopyableElement(I))
+ const bool IsCopyable = S.isCopyableElement(I);
+ if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost +=
- TTI.getArithmeticInstrCost(S.getOpcode(), I->getType(), CostKind);
+ if (IsCopyable || !I->isBinaryOp()) {
+ FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+ continue;
+ }
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ FMulPlusFAddCost += TTI.getArithmeticInstrCost(
+ I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
}
unsigned NumOps = 0;
for (auto [V, Op] : zip(VL, Operands.front())) {
>From 0d48cb7fdb01727ce595043fd896469f3d0c388a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 21 Aug 2026 23:37:03 +0200
Subject: [PATCH 12/13] only price add/sub-like lanes with
getArithmeticInstrCost
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 080da069d527d..a61742e7578f6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14510,7 +14510,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- if (IsCopyable || !I->isBinaryOp()) {
+ if (IsCopyable || (I->getOpcode() != S.getOpcode() &&
+ I->getOpcode() != S.getAltOpcode())) {
FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
continue;
}
>From 72df9911db26b00eda6cbeb76cd524855a8514f3 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 22 Aug 2026 00:52:01 +0200
Subject: [PATCH 13/13] apply review comment
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a61742e7578f6..a3490b5a3d0ba 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14502,6 +14502,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
};
FastMathFlags FMF;
FMF.set();
+ const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
+ S.isShiftOp() || S.isBitwiseLogicOp();
for (Value *V : VL) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
@@ -14510,8 +14512,9 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- if (IsCopyable || (I->getOpcode() != S.getOpcode() &&
- I->getOpcode() != S.getAltOpcode())) {
+ if (IsCopyable || !IsArithmeticState ||
+ (I->getOpcode() != S.getOpcode() &&
+ I->getOpcode() != S.getAltOpcode())) {
FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
continue;
}
More information about the llvm-commits
mailing list