[llvm] [SLP] Check the fmul's own flags in canConvertToFMA (PR #216599)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 16:03:09 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/216599
>From b747877bb9f6825d48f08f47045b5c9c726da1d9 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 16 Aug 2026 21:13:39 +0200
Subject: [PATCH 1/2] [SLP] Check the fmul's own flags in canConvertToFMA
CheckForContractable ran the operand bundle through the fadd's
InstructionsState, so every fmul in it failed the main/alt op test and was
skipped. The flags stayed set and allowContract was trivially true, which
priced a plain fmul feeding a contract fadd as if it would fuse. The backend
will not do that. Give the lambda the state to assess against and pass the
fmul's own.
Assisted-by: Claude Code Opus 5
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 7 +-
.../AArch64/recalc-copyable-node.ll | 61 ++--
.../SLPVectorizer/AArch64/wide-store.ll | 16 +-
.../AMDGPU/fma-operand-contract-selection.ll | 265 ++++++++++++++++++
.../SLPVectorizer/X86/dot-product.ll | 26 +-
5 files changed, 316 insertions(+), 59 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1b418ae5185ee..9ed5b62f27d31 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14400,7 +14400,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
"Can only convert to FMA for floating point types");
assert(S.isAddSubLikeOp() && "Can only convert to FMA for add/sub");
- auto CheckForContractable = [&](ArrayRef<Value *> VL) {
+ auto CheckForContractable = [](ArrayRef<Value *> VL,
+ const InstructionsState &S) {
FastMathFlags FMF;
FMF.set();
for (Value *V : VL) {
@@ -14417,7 +14418,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
}
return FMF.allowContract();
};
- if (!CheckForContractable(VL))
+ if (!CheckForContractable(VL, S))
return InstructionCost::getInvalid();
// fmul also should be contractable
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
@@ -14429,7 +14430,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
return InstructionCost::getInvalid();
- if (!CheckForContractable(Operands.front()))
+ if (!CheckForContractable(Operands.front(), OpS))
return InstructionCost::getInvalid();
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll
index 0500c00537a32..c88179da10110 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll
@@ -4,43 +4,38 @@
define { double, double } @test1(ptr %0, double %.unpack753, double %factor828) {
; CHECK-LABEL: define { double, double } @test1(
; CHECK-SAME: ptr [[TMP0:%.*]], double [[DOTUNPACK753:%.*]], double [[FACTOR828:%.*]]) {
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> <double poison, double 1.000000e+00>, double [[DOTUNPACK753]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
-; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x double> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP5]], <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> <double poison, double f0xC1502A98AE76C8B4>, double [[FACTOR828]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = fmul <2 x double> [[TMP9]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = fadd contract <2 x double> [[TMP11]], zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = fmul <2 x double> [[TMP12]], zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = fsub <2 x double> zeroinitializer, [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x double> [[TMP18]], <4 x double> <double 0.000000e+00, double poison, double 0.000000e+00, double poison>, <4 x i32> <i32 4, i32 0, i32 6, i32 1>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> <double poison, double poison, double -0.000000e+00, double poison>, <4 x double> [[TMP20]], <4 x i32> <i32 4, i32 5, i32 2, i32 poison>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 1>
-; CHECK-NEXT: [[TMP23:%.*]] = fadd contract <4 x double> [[TMP19]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x double> <double poison, double 1.000000e+00, double poison, double poison>, double [[DOTUNPACK753]], i64 0
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT: [[TMP26:%.*]] = fmul <4 x double> [[TMP23]], [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> <double 0.000000e+00, double poison, double poison, double poison>, <4 x i32> <i32 4, i32 0, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x double> [[TMP27]], <4 x double> [[TMP28]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT: [[TMP30:%.*]] = fadd <4 x double> [[TMP29]], [[TMP26]]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[TMP30]], i64 0
+; CHECK-NEXT: [[DOTELT752:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT: [[DOTUNPACK7532:%.*]] = load double, ptr [[DOTELT752]], align 8
+; CHECK-NEXT: [[DOTUNPACK1:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul double [[DOTUNPACK753]], [[DOTUNPACK1]]
+; CHECK-NEXT: [[FACTOR8283:%.*]] = fmul double [[TMP2]], 0.000000e+00
+; CHECK-NEXT: [[TMP3:%.*]] = fmul double [[FACTOR8283]], 0.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract double [[TMP3]], 0.000000e+00
+; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP4]], [[DOTUNPACK753]]
+; CHECK-NEXT: [[TMP31:%.*]] = fadd contract double [[TMP5]], 0.000000e+00
; CHECK-NEXT: [[TMP32:%.*]] = call { double, double } @__divdc3(double [[TMP31]])
+; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[DOTUNPACK1]], [[DOTUNPACK1]]
+; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], [[FACTOR828]]
+; CHECK-NEXT: [[TMP10:%.*]] = fadd contract double [[TMP9]], 0.000000e+00
+; CHECK-NEXT: [[TMP11:%.*]] = fmul double [[TMP10]], 0.000000e+00
; CHECK-NEXT: call void @llvm.stackrestore.p0(ptr null)
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x double> [[TMP30]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = fsub double 0.000000e+00, [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP12]], [[FACTOR8283]]
+; CHECK-NEXT: [[TMP14:%.*]] = fmul double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = fadd contract double [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = fmul double [[TMP15]], [[DOTUNPACK753]]
+; CHECK-NEXT: [[TMP33:%.*]] = fadd double [[DOTUNPACK753]], [[TMP16]]
; CHECK-NEXT: [[TMP34:%.*]] = call { double, double } @__divdc3(double [[TMP33]])
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[TMP30]], i64 2
+; CHECK-NEXT: [[TMP35:%.*]] = fadd double [[DOTUNPACK1]], 0.000000e+00
; CHECK-NEXT: [[TMP36:%.*]] = call { double, double } @__divdc3(double [[TMP35]])
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP30]], i64 3
+; CHECK-NEXT: [[TMP21:%.*]] = fmul double [[TMP2]], f0xC1502A98AE76C8B4
+; CHECK-NEXT: [[TMP22:%.*]] = fadd contract double [[TMP21]], 0.000000e+00
+; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00
+; CHECK-NEXT: [[TMP24:%.*]] = fsub double 0.000000e+00, [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = fmul double [[TMP24]], [[FACTOR8283]]
+; CHECK-NEXT: [[TMP26:%.*]] = fadd contract double [[TMP25]], [[TMP14]]
+; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], [[DOTUNPACK753]]
+; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[DOTUNPACK7532]], [[TMP27]]
; CHECK-NEXT: [[TMP38:%.*]] = call { double, double } @__divdc3(double [[TMP37]])
; CHECK-NEXT: ret { double, double } [[TMP38]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll
index 51c2040a86d64..37fc33318430b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll
@@ -5,20 +5,22 @@ define void @test(ptr %p, float %conv, i64 %n) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[P:%.*]], float [[CONV:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[CONV]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP:.*]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[OUT:%.*]] = phi ptr [ [[P]], %[[ENTRY]] ], [ [[OUT_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[M2:%.*]] = fmul float [[CONV]], 0.000000e+00
-; CHECK-NEXT: [[M3:%.*]] = fmul float [[CONV]], 0.000000e+00
; CHECK-NEXT: [[S0:%.*]] = fsub float 0.000000e+00, [[CONV]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> <float poison, float poison, float poison, float -0.000000e+00>, float [[S0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[M2]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[M3]], i64 2
-; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <4 x float> <float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, [[TMP2]]
-; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[OUT]], align 4
+; CHECK-NEXT: store float [[S0]], ptr [[OUT]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[O1:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 4
+; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <2 x float> [[TMP2]], zeroinitializer
+; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[O1]], align 4
+; CHECK-NEXT: [[O3:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 12
+; CHECK-NEXT: store float 0.000000e+00, ptr [[O3]], align 4
; CHECK-NEXT: [[OUT_NEXT]] = getelementptr inbounds i8, ptr [[OUT]], i64 16
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
new file mode 100644
index 0000000000000..9723bdd6bb369
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
@@ -0,0 +1,265 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=15 < %s | FileCheck %s --check-prefix=THR15
+
+; The fma check has to read the multiply's own fast math flags. Reading the
+; add's instead makes every multiply look contractable, so a fusion that the
+; backend will not perform gets paid for. mixed_contract_mul carries contract
+; on one lane only and moves at the default threshold.
+
+
+define void @nocontract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @nocontract_mul(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+; THR15-LABEL: define void @nocontract_mul(
+; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; THR15-NEXT: [[ENTRY:.*:]]
+; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
+; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
+; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
+; THR15-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
+; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
+; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4
+; THR15-NEXT: ret void
+;
+entry:
+ %ap0 = getelementptr inbounds float, ptr %a, i64 0
+ %bp0 = getelementptr inbounds float, ptr %b, i64 0
+ %cp0 = getelementptr inbounds float, ptr %c, i64 0
+ %dp0 = getelementptr inbounds float, ptr %d, i64 0
+ %a0 = load float, ptr %ap0
+ %b0 = load float, ptr %bp0
+ %c0 = load float, ptr %cp0
+ %m0 = fmul float %a0, %b0
+ %r0 = fadd contract float %m0, %c0
+ store float %r0, ptr %dp0
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ %a1 = load float, ptr %ap1
+ %b1 = load float, ptr %bp1
+ %c1 = load float, ptr %cp1
+ %m1 = fmul float %a1, %b1
+ %r1 = fadd contract float %m1, %c1
+ store float %r1, ptr %dp1
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ %a2 = load float, ptr %ap2
+ %b2 = load float, ptr %bp2
+ %c2 = load float, ptr %cp2
+ %m2 = fmul float %a2, %b2
+ %r2 = fadd contract float %m2, %c2
+ store float %r2, ptr %dp2
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ %a3 = load float, ptr %ap3
+ %b3 = load float, ptr %bp3
+ %c3 = load float, ptr %cp3
+ %m3 = fmul float %a3, %b3
+ %r3 = fadd contract float %m3, %c3
+ store float %r3, ptr %dp3
+ ret void
+}
+
+define void @contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @contract_mul(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+; THR15-LABEL: define void @contract_mul(
+; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; THR15-NEXT: [[ENTRY:.*:]]
+; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
+; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
+; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
+; THR15-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP0]], [[TMP1]]
+; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
+; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4
+; THR15-NEXT: ret void
+;
+entry:
+ %ap0 = getelementptr inbounds float, ptr %a, i64 0
+ %bp0 = getelementptr inbounds float, ptr %b, i64 0
+ %cp0 = getelementptr inbounds float, ptr %c, i64 0
+ %dp0 = getelementptr inbounds float, ptr %d, i64 0
+ %a0 = load float, ptr %ap0
+ %b0 = load float, ptr %bp0
+ %c0 = load float, ptr %cp0
+ %m0 = fmul contract float %a0, %b0
+ %r0 = fadd contract float %m0, %c0
+ store float %r0, ptr %dp0
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ %a1 = load float, ptr %ap1
+ %b1 = load float, ptr %bp1
+ %c1 = load float, ptr %cp1
+ %m1 = fmul contract float %a1, %b1
+ %r1 = fadd contract float %m1, %c1
+ store float %r1, ptr %dp1
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ %a2 = load float, ptr %ap2
+ %b2 = load float, ptr %bp2
+ %c2 = load float, ptr %cp2
+ %m2 = fmul contract float %a2, %b2
+ %r2 = fadd contract float %m2, %c2
+ store float %r2, ptr %dp2
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ %a3 = load float, ptr %ap3
+ %b3 = load float, ptr %bp3
+ %c3 = load float, ptr %cp3
+ %m3 = fmul contract float %a3, %b3
+ %r3 = fadd contract float %m3, %c3
+ store float %r3, ptr %dp3
+ ret void
+}
+
+define void @mixed_contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @mixed_contract_mul(
+; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4
+; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
+; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4
+; CHECK-NEXT: ret void
+;
+; THR15-LABEL: define void @mixed_contract_mul(
+; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; THR15-NEXT: [[ENTRY:.*:]]
+; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
+; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
+; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
+; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
+; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
+; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
+; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
+; THR15-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
+; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
+; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4
+; THR15-NEXT: ret void
+;
+entry:
+ %ap0 = getelementptr inbounds float, ptr %a, i64 0
+ %bp0 = getelementptr inbounds float, ptr %b, i64 0
+ %cp0 = getelementptr inbounds float, ptr %c, i64 0
+ %dp0 = getelementptr inbounds float, ptr %d, i64 0
+ %a0 = load float, ptr %ap0
+ %b0 = load float, ptr %bp0
+ %c0 = load float, ptr %cp0
+ %m0 = fmul contract float %a0, %b0
+ %r0 = fadd contract float %m0, %c0
+ store float %r0, ptr %dp0
+ %ap1 = getelementptr inbounds float, ptr %a, i64 1
+ %bp1 = getelementptr inbounds float, ptr %b, i64 1
+ %cp1 = getelementptr inbounds float, ptr %c, i64 1
+ %dp1 = getelementptr inbounds float, ptr %d, i64 1
+ %a1 = load float, ptr %ap1
+ %b1 = load float, ptr %bp1
+ %c1 = load float, ptr %cp1
+ %m1 = fmul float %a1, %b1
+ %r1 = fadd contract float %m1, %c1
+ store float %r1, ptr %dp1
+ %ap2 = getelementptr inbounds float, ptr %a, i64 2
+ %bp2 = getelementptr inbounds float, ptr %b, i64 2
+ %cp2 = getelementptr inbounds float, ptr %c, i64 2
+ %dp2 = getelementptr inbounds float, ptr %d, i64 2
+ %a2 = load float, ptr %ap2
+ %b2 = load float, ptr %bp2
+ %c2 = load float, ptr %cp2
+ %m2 = fmul float %a2, %b2
+ %r2 = fadd contract float %m2, %c2
+ store float %r2, ptr %dp2
+ %ap3 = getelementptr inbounds float, ptr %a, i64 3
+ %bp3 = getelementptr inbounds float, ptr %b, i64 3
+ %cp3 = getelementptr inbounds float, ptr %c, i64 3
+ %dp3 = getelementptr inbounds float, ptr %d, i64 3
+ %a3 = load float, ptr %ap3
+ %b3 = load float, ptr %bp3
+ %c3 = load float, ptr %cp3
+ %m3 = fmul float %a3, %b3
+ %r3 = fadd contract float %m3, %c3
+ store float %r3, ptr %dp3
+ ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
index c499aea3819c7..ef1da6fbd8ca2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
@@ -363,14 +363,11 @@ define double @dot2f64_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(1
; AVX-NEXT: ret double [[DOT01]]
;
; AVX2-LABEL: @dot2f64_fast(
-; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1
-; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1
-; AVX2-NEXT: [[X0:%.*]] = load double, ptr [[PTRX]], align 4
-; AVX2-NEXT: [[Y0:%.*]] = load double, ptr [[PTRY]], align 4
-; AVX2-NEXT: [[X1:%.*]] = load double, ptr [[PTRX1]], align 4
-; AVX2-NEXT: [[Y1:%.*]] = load double, ptr [[PTRY1]], align 4
-; AVX2-NEXT: [[MUL0:%.*]] = fmul double [[X0]], [[Y0]]
-; AVX2-NEXT: [[MUL1:%.*]] = fmul double [[X1]], [[Y1]]
+; AVX2-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4
+; AVX2-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4
+; AVX2-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]
+; AVX2-NEXT: [[MUL0:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; AVX2-NEXT: [[MUL1:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
; AVX2-NEXT: [[DOT01:%.*]] = fadd fast double [[MUL0]], [[MUL1]]
; AVX2-NEXT: ret double [[DOT01]]
;
@@ -413,14 +410,11 @@ define float @dot2f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16
; AVX-NEXT: ret float [[DOT01]]
;
; AVX2-LABEL: @dot2f32_fast(
-; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1
-; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1
-; AVX2-NEXT: [[X0:%.*]] = load float, ptr [[PTRX]], align 4
-; AVX2-NEXT: [[Y0:%.*]] = load float, ptr [[PTRY]], align 4
-; AVX2-NEXT: [[X1:%.*]] = load float, ptr [[PTRX1]], align 4
-; AVX2-NEXT: [[Y1:%.*]] = load float, ptr [[PTRY1]], align 4
-; AVX2-NEXT: [[MUL0:%.*]] = fmul float [[X0]], [[Y0]]
-; AVX2-NEXT: [[MUL1:%.*]] = fmul float [[X1]], [[Y1]]
+; AVX2-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4
+; AVX2-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4
+; AVX2-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]
+; AVX2-NEXT: [[MUL0:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
+; AVX2-NEXT: [[MUL1:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
; AVX2-NEXT: [[DOT01:%.*]] = fadd fast float [[MUL0]], [[MUL1]]
; AVX2-NEXT: ret float [[DOT01]]
;
>From 14423e79d16ca40deb06f39b21b67d8c553c067e Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 17 Aug 2026 00:46:12 +0200
Subject: [PATCH 2/2] Regenerate recalc-copyable-deps-on-reorder.ll
---
.../recalc-copyable-deps-on-reorder.ll | 124 ++++++++----------
1 file changed, 56 insertions(+), 68 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
index f7b2664e57b80..5bad17fa9c87d 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
@@ -7,89 +7,77 @@ define void @test(ptr %0, ptr %1, double %2, ptr %3) {
; CHECK-NEXT: br i1 true, label %[[VECTOR_PH:.*]], label %[[DOTPREHEADER511:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[TMP7:%.*]] = fadd double [[TMP6]], 0.000000e+00
-; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[TMP6]], 0.000000e+00
-; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00
; CHECK-NEXT: [[TMP10:%.*]] = fadd double [[TMP2]], 0.000000e+00
; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 1.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fadd double [[TMP9]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP2]], 0.000000e+00
-; CHECK-NEXT: [[TMP14:%.*]] = fadd contract double [[TMP13]], 0.000000e+00
-; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP6]], [[TMP2]]
-; CHECK-NEXT: [[TMP16:%.*]] = fadd contract double [[TMP15]], 0.000000e+00
; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP17]], 0.000000e+00
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = fmul <2 x double> [[TMP20]], zeroinitializer
; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP20:%.*]] = fmul double [[TMP2]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = fadd double [[TMP20]], 0.000000e+00
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP12]], zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> [[TMP12]], double [[TMP19]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP36:%.*]] = fmul <2 x double> [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[TMP49:%.*]] = fadd <2 x double> [[TMP36]], zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00
-; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP2]], [[TMP23]]
-; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP24]], 0.000000e+00
-; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], 0.000000e+00
-; CHECK-NEXT: [[TMP28:%.*]] = fadd double [[TMP27]], 0.000000e+00
-; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP2]], [[TMP17]]
-; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x double> [[TMP12]], double [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP68:%.*]] = fmul <2 x double> [[TMP21]], zeroinitializer
+; CHECK-NEXT: [[TMP69:%.*]] = fmul <2 x double> [[TMP68]], zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP11]], i64 0
+; CHECK-NEXT: [[TMP25:%.*]] = fadd <2 x double> [[TMP69]], [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x double> poison, double [[TMP23]], i64 0
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x double> [[TMP28]], double [[TMP17]], i64 1
+; CHECK-NEXT: [[TMP30:%.*]] = fmul <2 x double> [[TMP27]], [[TMP29]]
+; CHECK-NEXT: [[TMP31:%.*]] = fadd <2 x double> [[TMP30]], zeroinitializer
; CHECK-NEXT: br label %[[DOTPREHEADER511]]
; CHECK: [[_PREHEADER511:.*:]]
-; CHECK-NEXT: [[DOTSROA_9_0:%.*]] = phi double [ [[TMP30]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4:%.*]] ]
-; CHECK-NEXT: [[DOTSROA_7_0:%.*]] = phi double [ [[TMP28]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_6887_0:%.*]] = phi double [ [[TMP25]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_5_0:%.*]] = phi double [ [[TMP21]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_4886_0:%.*]] = phi double [ [[TMP16]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_3_0:%.*]] = phi double [ [[TMP14]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_2885_0:%.*]] = phi double [ [[TMP12]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
-; CHECK-NEXT: [[DOTSROA_0884_0:%.*]] = phi double [ [[TMP7]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x double> [ [[TMP14]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4:%.*]] ]
+; CHECK-NEXT: [[TMP33:%.*]] = phi <2 x double> [ [[TMP49]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ]
+; CHECK-NEXT: [[TMP34:%.*]] = phi <2 x double> [ [[TMP31]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ]
+; CHECK-NEXT: [[TMP35:%.*]] = phi <2 x double> [ [[TMP25]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ]
; CHECK-NEXT: br i1 true, label %[[BB31:.*]], [[DOTPREHEADER498:label %.*]]
; CHECK: [[BB31]]:
-; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
-; CHECK-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP3]], align 8
-; CHECK-NEXT: [[TMP34:%.*]] = fmul double [[DOTSROA_0884_0]], [[TMP33]]
-; CHECK-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP32]], align 8
-; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[TMP34]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = fmul double [[DOTSROA_4886_0]], 0.000000e+00
-; CHECK-NEXT: [[TMP39:%.*]] = fadd double [[TMP37]], [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], 0.000000e+00
-; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], 1.000000e+00
-; CHECK-NEXT: [[TMP42:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP33]]
-; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[DOTSROA_3_0]], [[TMP35]]
-; CHECK-NEXT: [[TMP44:%.*]] = fadd double [[TMP42]], [[TMP43]]
-; CHECK-NEXT: [[TMP45:%.*]] = fmul double [[DOTSROA_5_0]], 0.000000e+00
-; CHECK-NEXT: [[TMP46:%.*]] = fadd double [[TMP44]], [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = fadd double [[TMP46]], [[TMP2]]
-; CHECK-NEXT: [[TMP48:%.*]] = fadd double [[TMP47]], 0.000000e+00
-; CHECK-NEXT: [[TMP49:%.*]] = fmul double [[DOTSROA_4886_0]], [[TMP33]]
-; CHECK-NEXT: [[TMP50:%.*]] = fmul double [[DOTSROA_5_0]], [[TMP35]]
-; CHECK-NEXT: [[TMP51:%.*]] = fadd double [[TMP49]], [[TMP50]]
+; CHECK-NEXT: [[TMP37:%.*]] = fmul <2 x double> [[TMP33]], zeroinitializer
+; CHECK-NEXT: [[TMP38:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <2 x double> [[TMP35]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP40:%.*]] = fmul <2 x double> [[TMP39]], [[TMP38]]
+; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x double> [[TMP40]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP42:%.*]] = fmul <2 x double> [[TMP32]], [[TMP38]]
+; CHECK-NEXT: [[TMP43:%.*]] = fadd <2 x double> [[TMP41]], [[TMP42]]
+; CHECK-NEXT: [[TMP44:%.*]] = fadd <2 x double> [[TMP43]], [[TMP37]]
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP46:%.*]] = fadd <2 x double> [[TMP44]], [[TMP45]]
+; CHECK-NEXT: [[TMP47:%.*]] = fadd <2 x double> [[TMP46]], <double 1.000000e+00, double 0.000000e+00>
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP38]], i64 0
; CHECK-NEXT: [[TMP52:%.*]] = load double, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP53:%.*]] = fmul double [[DOTSROA_6887_0]], [[TMP52]]
-; CHECK-NEXT: [[TMP54:%.*]] = fadd double [[TMP51]], [[TMP53]]
-; CHECK-NEXT: [[TMP55:%.*]] = fmul double [[DOTSROA_9_0]], [[TMP2]]
-; CHECK-NEXT: [[TMP56:%.*]] = fadd double [[TMP54]], [[TMP55]]
-; CHECK-NEXT: [[TMP57:%.*]] = fadd double [[TMP56]], 0.000000e+00
-; CHECK-NEXT: [[TMP58:%.*]] = fmul double [[DOTSROA_7_0]], [[TMP33]]
-; CHECK-NEXT: [[TMP59:%.*]] = fadd double [[TMP58]], [[TMP2]]
-; CHECK-NEXT: [[TMP60:%.*]] = fmul double [[DOTSROA_9_0]], 0.000000e+00
-; CHECK-NEXT: [[TMP61:%.*]] = fadd double [[TMP59]], [[TMP60]]
-; CHECK-NEXT: [[TMP62:%.*]] = fadd double [[TMP61]], 0.000000e+00
-; CHECK-NEXT: [[TMP63:%.*]] = fadd double [[TMP62]], 1.000000e+00
+; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> [[TMP34]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <2 x double> [[TMP51]], double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP53:%.*]] = fmul <2 x double> [[TMP50]], [[TMP70]]
+; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> [[TMP35]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP56:%.*]] = fmul <2 x double> [[TMP54]], [[TMP55]]
+; CHECK-NEXT: [[TMP57:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <2 x double> [[TMP57]], <2 x double> [[TMP53]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP59:%.*]] = fadd <2 x double> [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[TMP60:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP52]], i64 0
+; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP34]], [[TMP60]]
+; CHECK-NEXT: [[TMP62:%.*]] = fadd <2 x double> [[TMP59]], [[TMP61]]
+; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <2 x double> <double poison, double 0.000000e+00>, <2 x double> [[TMP53]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT: [[TMP64:%.*]] = fadd <2 x double> [[TMP62]], [[TMP63]]
+; CHECK-NEXT: [[TMP65:%.*]] = fadd <2 x double> [[TMP64]], <double 0.000000e+00, double 1.000000e+00>
; CHECK-NEXT: br [[DOTPREHEADER498]]
; CHECK: [[_PREHEADER498:.*:]]
-; CHECK-NEXT: [[DOTSROA_0913_0:%.*]] = phi double [ [[TMP41]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
-; CHECK-NEXT: [[DOTSROA_6915_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
-; CHECK-NEXT: [[DOTSROA_10918_0:%.*]] = phi double [ [[TMP57]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
-; CHECK-NEXT: [[DOTSROA_14921_0:%.*]] = phi double [ [[TMP63]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
-; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP33]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
-; CHECK-NEXT: store double [[DOTSROA_0913_0]], ptr [[TMP1]], align 8
-; CHECK-NEXT: [[DOTSROA_6915_0__SROA_IDX916:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8
-; CHECK-NEXT: store double [[DOTSROA_6915_0]], ptr [[DOTSROA_6915_0__SROA_IDX916]], align 8
+; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[TMP66:%.*]] = phi <2 x double> [ [[TMP47]], %[[BB31]] ], [ zeroinitializer, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[TMP67:%.*]] = phi <2 x double> [ [[TMP65]], %[[BB31]] ], [ zeroinitializer, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: store <2 x double> [[TMP66]], ptr [[TMP1]], align 8
; CHECK-NEXT: [[DOTSROA_10918_0__SROA_IDX919:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
-; CHECK-NEXT: store double [[DOTSROA_10918_0]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8
-; CHECK-NEXT: [[DOTSROA_14921_0__SROA_IDX922:%.*]] = getelementptr i8, ptr [[TMP1]], i64 24
-; CHECK-NEXT: store double [[DOTSROA_14921_0]], ptr [[DOTSROA_14921_0__SROA_IDX922]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP67]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8
; CHECK-NEXT: store double [[DOTSROA_18924_0]], ptr [[TMP0]], align 8
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list