[llvm] [SLP] Account for fma fusion when vectorizing an ordered fadd reduction (PR #210399)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 09:49:34 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/210399
>From 101f315e579ca0278f68eb8079aa29af23423bc8 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 17 Jul 2026 19:45:50 +0200
Subject: [PATCH 1/8] [SLP] Account for lost fma fusion when vectorizing an
ordered fadd reduction
An ordered fadd reduction of contractable fmuls lowers to a chain of fmas in
scalar code. SLP vectorizes the fmul operand tree and leaves the fadds as an
ordered scalar chain, which breaks that fusion, but the ordered-reduction cost
path hardcodes ReductionCost = 0 and never accounts for it. Charge the fusion
saving per reduced fmul so the cost reflects the lost fusion.
The saving is zero on targets without a faster fma, so only targets that
actually fuse are affected.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 +++++++++++-
.../SLPVectorizer/X86/slp-fma-loss-ordered.ll | 40 +++++++++++++++++++
2 files changed, 63 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e22ba3bd149c..68d75fa366bc1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30217,9 +30217,30 @@ class HorizontalReduction {
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
- V.isReducedCmpBitcastRoot())
+ V.isReducedCmpBitcastRoot()) {
ReductionCost = 0;
- else
+ // Check for potential fma fusion as vectorization would break it.
+ if (RdxKind == RecurKind::FAdd && RdxFMF.allowContract()) {
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ Type *Ty = VL.front()->getType();
+ IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
+ RdxFMF);
+ InstructionCost FusionSaving =
+ TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
+ TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
+ TTI->getIntrinsicInstrCost(ICA, CostKind);
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (FMul && FMul->getOpcode() == Instruction::FMul &&
+ FMul->hasOneUse() &&
+ cast<FPMathOperator>(FMul)
+ ->getFastMathFlags()
+ .allowContract())
+ ReductionCost += FusionSaving;
+ }
+ }
+ } else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
RdxFMF, V, DT, DL, TLI);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
new file mode 100644
index 0000000000000..2aefd208e3a7f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=NOFMA
+; RUN: opt -passes=slp-vectorizer -S -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=FMA
+
+; On a target with fast fma changing chain of fmul into fmul is not profitable
+; when fusion of fmul -> fadd sequence in fma is possible, so the reduction
+; should stay scalar.
+
+define double @mul_fun() {
+; NOFMA-LABEL: @mul_fun(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = call contract double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP4]]
+;
+; FMA-LABEL: @mul_fun(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[MUL0:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[MUL0]], [[CVT0]]
+; FMA-NEXT: [[MUL1:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
+; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[MUL1]], [[ADD0]]
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
+; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul contract double 7.000000e+00, %cvt0
+ %add0 = fadd contract double %mul0, %cvt0
+ %mul1 = fmul contract double -4.300000e+01, %cvt0
+ %add1 = fadd contract double %mul1, %add0
+ %mul2 = fmul contract double 2.200000e-02, %cvt0
+ %add2 = fadd contract double %mul2, %add1
+ %mul3 = fmul contract double 9.500000e+00, %cvt0
+ %add3 = fadd contract double %mul3, %add2
+ ret double %add3
+}
>From 8f5f1e560f643280867ea9a5f4adc881a6f528f8 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 17 Jul 2026 23:21:46 +0200
Subject: [PATCH 2/8] review comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 36 ++++-----
.../SLPVectorizer/X86/slp-fma-loss-ordered.ll | 80 +++++++++++++++++++
2 files changed, 98 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 68d75fa366bc1..acdfb2bc83f9d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30220,25 +30220,25 @@ class HorizontalReduction {
V.isReducedCmpBitcastRoot()) {
ReductionCost = 0;
// Check for potential fma fusion as vectorization would break it.
- if (RdxKind == RecurKind::FAdd && RdxFMF.allowContract()) {
+ if (RK == ReductionOrdering::Ordered && RdxKind == RecurKind::FAdd &&
+ RdxFMF.allowContract()) {
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- Type *Ty = VL.front()->getType();
- IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
- RdxFMF);
- InstructionCost FusionSaving =
- TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
- TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
- TTI->getIntrinsicInstrCost(ICA, CostKind);
- if (FusionSaving.isValid() && FusionSaving > 0)
- for (Value *RdxVal : VL) {
- auto *FMul = dyn_cast<Instruction>(RdxVal);
- if (FMul && FMul->getOpcode() == Instruction::FMul &&
- FMul->hasOneUse() &&
- cast<FPMathOperator>(FMul)
- ->getFastMathFlags()
- .allowContract())
- ReductionCost += FusionSaving;
- }
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse())
+ continue;
+ auto *FAdd = cast<Instruction>(FMul->user_back());
+ InstructionCost FMACost = canConvertToFMA(
+ FAdd, InstructionsState(FAdd, FAdd), DT, DL, *TTI, TLI);
+ if (!FMACost.isValid())
+ continue;
+ InstructionCost FusionSaving =
+ TTI->getInstructionCost(FMul, CostKind) +
+ TTI->getInstructionCost(FAdd, CostKind) - FMACost;
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ ReductionCost += FusionSaving;
+ }
}
} else
ReductionCost =
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
index 2aefd208e3a7f..642d308d20feb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -38,3 +38,83 @@ define double @mul_fun() {
%add3 = fadd contract double %mul3, %add2
ret double %add3
}
+
+; %mul1 keeps a second use, so fusion was never on the table for it, it must not
+; be penalized even on an fma-capable target.
+define double @mul_fun_multiuse(ptr %dst) {
+; NOFMA-LABEL: @mul_fun_multiuse(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; NOFMA-NEXT: store double [[TMP4]], ptr [[DST:%.*]], align 8
+; NOFMA-NEXT: [[TMP5:%.*]] = call contract double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP5]]
+;
+; FMA-LABEL: @mul_fun_multiuse(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[CVT0]], i64 0
+; FMA-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
+; FMA-NEXT: [[TMP3:%.*]] = fmul contract <2 x double> <double -4.300000e+01, double 7.000000e+00>, [[TMP2]]
+; FMA-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[TMP4]], [[CVT0]]
+; FMA-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; FMA-NEXT: store double [[TMP5]], ptr [[DST:%.*]], align 8
+; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[TMP5]], [[ADD0]]
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
+; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul contract double 7.000000e+00, %cvt0
+ %add0 = fadd contract double %mul0, %cvt0
+ %mul1 = fmul contract double -4.300000e+01, %cvt0
+ store double %mul1, ptr %dst
+ %add1 = fadd contract double %mul1, %add0
+ %mul2 = fmul contract double 2.200000e-02, %cvt0
+ %add2 = fadd contract double %mul2, %add1
+ %mul3 = fmul contract double 9.500000e+00, %cvt0
+ %add3 = fadd contract double %mul3, %add2
+ ret double %add3
+}
+
+; No contract flag anywhere, so fusion was never possible. Check that vectorization
+; has happened.
+define double @mul_fun_no_contract() {
+; NOFMA-LABEL: @mul_fun_no_contract(
+; NOFMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; NOFMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; NOFMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; NOFMA-NEXT: [[TMP3:%.*]] = fmul <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; NOFMA-NEXT: [[TMP4:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double [[CVT0]], <4 x double> [[TMP3]])
+; NOFMA-NEXT: ret double [[TMP4]]
+;
+; FMA-LABEL: @mul_fun_no_contract(
+; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
+; FMA-NEXT: [[MUL0:%.*]] = fmul double 7.000000e+00, [[CVT0]]
+; FMA-NEXT: [[ADD0:%.*]] = fadd double [[MUL0]], [[CVT0]]
+; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
+; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; FMA-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
+; FMA-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[ADD1:%.*]] = fadd double [[TMP4]], [[ADD0]]
+; FMA-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[ADD2:%.*]] = fadd double [[TMP5]], [[ADD1]]
+; FMA-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[ADD3:%.*]] = fadd double [[TMP6]], [[ADD2]]
+; FMA-NEXT: ret double [[ADD3]]
+;
+ %cvt0 = uitofp i16 3 to double
+ %mul0 = fmul double 7.000000e+00, %cvt0
+ %add0 = fadd double %mul0, %cvt0
+ %mul1 = fmul double -4.300000e+01, %cvt0
+ %add1 = fadd double %mul1, %add0
+ %mul2 = fmul double 2.200000e-02, %cvt0
+ %add2 = fadd double %mul2, %add1
+ %mul3 = fmul double 9.500000e+00, %cvt0
+ %add3 = fadd double %mul3, %add2
+ ret double %add3
+}
>From e878bb5dccf2d41710c3e25fd00b99ad5ccabd20 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 18 Jul 2026 01:57:21 +0200
Subject: [PATCH 3/8] style
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c04ee0402dd8a..7ef232596b10f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30544,10 +30544,11 @@ class HorizontalReduction {
ReductionCost += FusionSaving;
}
}
- } else
+ } else {
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
RdxFMF, V, DT, DL, TLI);
+ }
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
>From 75007ee3dfb3a2640a468fb6f3fd9f1256858a03 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 20 Jul 2026 10:33:34 +0200
Subject: [PATCH 4/8] use per-type cost model
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 31 +++++++++----------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7ef232596b10f..740a5ac4b5a75 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30527,22 +30527,21 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered && RdxKind == RecurKind::FAdd &&
RdxFMF.allowContract()) {
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- for (Value *RdxVal : VL) {
- auto *FMul = dyn_cast<Instruction>(RdxVal);
- if (!FMul || FMul->getOpcode() != Instruction::FMul ||
- !FMul->hasOneUse())
- continue;
- auto *FAdd = cast<Instruction>(FMul->user_back());
- InstructionCost FMACost = canConvertToFMA(
- FAdd, InstructionsState(FAdd, FAdd), DT, DL, *TTI, TLI);
- if (!FMACost.isValid())
- continue;
- InstructionCost FusionSaving =
- TTI->getInstructionCost(FMul, CostKind) +
- TTI->getInstructionCost(FAdd, CostKind) - FMACost;
- if (FusionSaving.isValid() && FusionSaving > 0)
- ReductionCost += FusionSaving;
- }
+ Type *Ty = VL.front()->getType();
+ IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
+ RdxFMF);
+ InstructionCost FusionSaving =
+ TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
+ TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
+ TTI->getIntrinsicInstrCost(ICA, CostKind);
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (FMul && FMul->getOpcode() == Instruction::FMul &&
+ FMul->hasOneUse() &&
+ cast<FPMathOperator>(FMul)->getFastMathFlags().allowContract())
+ ReductionCost += FusionSaving;
+ }
}
} else {
ReductionCost =
>From 5de6b16ebfb996fbf6db868d17c99b882653aa63 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 21 Jul 2026 19:19:31 +0200
Subject: [PATCH 5/8] regenerate tests
---
.../AMDGPU/ordered-reduction-fma-fusion.ll | 196 +++++++++++-------
.../NVPTX/ordered-reduction-fma-fusion.ll | 12 +-
2 files changed, 124 insertions(+), 84 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index cdd9fcea22d3f..ed0d3b8dae293 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -9,103 +9,139 @@
define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-LABEL: @conv_contract(
; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
+; CHECK-NEXT: [[IV0:%.*]] = load float, ptr addrspace(1) [[IP0]], align 4
; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
+; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP0]], align 4
+; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV0]], [[MV0]]
+; CHECK-NEXT: [[IP1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1
+; CHECK-NEXT: [[IV1:%.*]] = load float, ptr addrspace(1) [[IP1]], align 4
+; CHECK-NEXT: [[MP1:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1
+; CHECK-NEXT: [[MV1:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4
+; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[IV1]], [[MV1]]
+; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[PROD1]]
+; CHECK-NEXT: [[IP2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 2
+; CHECK-NEXT: [[IV2:%.*]] = load float, ptr addrspace(1) [[IP2]], align 4
+; CHECK-NEXT: [[MP2:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 2
+; CHECK-NEXT: [[MV2:%.*]] = load float, ptr addrspace(4) [[MP2]], align 4
+; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[IV2]], [[MV2]]
+; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
+; CHECK-NEXT: [[IP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 3
+; CHECK-NEXT: [[IV3:%.*]] = load float, ptr addrspace(1) [[IP3]], align 4
+; CHECK-NEXT: [[MP3:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 3
+; CHECK-NEXT: [[MV3:%.*]] = load float, ptr addrspace(4) [[MP3]], align 4
+; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[IV3]], [[MV3]]
+; CHECK-NEXT: [[ACC3:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
; CHECK-NEXT: [[IV4:%.*]] = load float, ptr addrspace(1) [[IP4]], align 4
+; CHECK-NEXT: [[MP4:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 4
+; CHECK-NEXT: [[MV4:%.*]] = load float, ptr addrspace(4) [[MP4]], align 4
+; CHECK-NEXT: [[PROD4:%.*]] = fmul contract float [[IV4]], [[MV4]]
+; CHECK-NEXT: [[ACC4:%.*]] = fadd contract float [[ACC3]], [[PROD4]]
; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4
+; CHECK-NEXT: [[MP5:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 5
+; CHECK-NEXT: [[MV5:%.*]] = load float, ptr addrspace(4) [[MP5]], align 4
+; CHECK-NEXT: [[PROD5:%.*]] = fmul contract float [[IV5]], [[MV5]]
+; CHECK-NEXT: [[ACC5:%.*]] = fadd contract float [[ACC4]], [[PROD5]]
; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
+; CHECK-NEXT: [[IV6:%.*]] = load float, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[MP6:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 6
+; CHECK-NEXT: [[MV6:%.*]] = load float, ptr addrspace(4) [[MP6]], align 4
+; CHECK-NEXT: [[PROD6:%.*]] = fmul contract float [[IV6]], [[MV6]]
+; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC5]], [[PROD6]]
+; CHECK-NEXT: [[IP7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 10
+; CHECK-NEXT: [[IV7:%.*]] = load float, ptr addrspace(1) [[IP7]], align 4
+; CHECK-NEXT: [[MP7:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 7
+; CHECK-NEXT: [[MV7:%.*]] = load float, ptr addrspace(4) [[MP7]], align 4
+; CHECK-NEXT: [[PROD7:%.*]] = fmul contract float [[IV7]], [[MV7]]
+; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[PROD7]]
; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
+; CHECK-NEXT: [[IV8:%.*]] = load float, ptr addrspace(1) [[IP8]], align 4
+; CHECK-NEXT: [[MP8:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 8
+; CHECK-NEXT: [[MV8:%.*]] = load float, ptr addrspace(4) [[MP8]], align 4
+; CHECK-NEXT: [[PROD8:%.*]] = fmul contract float [[IV8]], [[MV8]]
+; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[PROD8]]
+; CHECK-NEXT: [[IP9:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12
+; CHECK-NEXT: [[IV9:%.*]] = load float, ptr addrspace(1) [[IP9]], align 4
+; CHECK-NEXT: [[MP9:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 9
+; CHECK-NEXT: [[MV9:%.*]] = load float, ptr addrspace(4) [[MP9]], align 4
+; CHECK-NEXT: [[PROD9:%.*]] = fmul contract float [[IV9]], [[MV9]]
+; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[PROD9]]
; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
+; CHECK-NEXT: [[IV10:%.*]] = load float, ptr addrspace(1) [[IP10]], align 4
+; CHECK-NEXT: [[MP10:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 10
+; CHECK-NEXT: [[MV10:%.*]] = load float, ptr addrspace(4) [[MP10]], align 4
+; CHECK-NEXT: [[PROD10:%.*]] = fmul contract float [[IV10]], [[MV10]]
+; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[PROD10]]
+; CHECK-NEXT: [[IP11:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17
+; CHECK-NEXT: [[IV11:%.*]] = load float, ptr addrspace(1) [[IP11]], align 4
+; CHECK-NEXT: [[MP11:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 11
+; CHECK-NEXT: [[MV11:%.*]] = load float, ptr addrspace(4) [[MP11]], align 4
+; CHECK-NEXT: [[PROD11:%.*]] = fmul contract float [[IV11]], [[MV11]]
+; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[PROD11]]
; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
-; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20
-; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4
+; CHECK-NEXT: [[IV12:%.*]] = load float, ptr addrspace(1) [[IP12]], align 4
+; CHECK-NEXT: [[MP12:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 12
+; CHECK-NEXT: [[MV12:%.*]] = load float, ptr addrspace(4) [[MP12]], align 4
+; CHECK-NEXT: [[PROD12:%.*]] = fmul contract float [[IV12]], [[MV12]]
+; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[PROD12]]
+; CHECK-NEXT: [[IP13:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19
+; CHECK-NEXT: [[MP13:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 13
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[IP13]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(4) [[MP13]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
+; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
+; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP5]]
; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV4]], i64 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV5]], i64 5
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <16 x float> [[TMP8]], float [[IV14]], i64 14
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[IV15]], i64 15
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP24]], <16 x float> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP20]], [[TMP6]]
-; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
-; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
-; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[PROD15]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
-; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
-; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP21]], i64 4
-; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[TMP14]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP21]], i64 5
-; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[TMP15]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x float> [[TMP21]], i64 6
-; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC19]], [[TMP28]]
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <16 x float> [[TMP21]], i64 7
-; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[TMP29]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <16 x float> [[TMP21]], i64 8
-; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[TMP30]]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <16 x float> [[TMP21]], i64 9
-; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[TMP31]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <16 x float> [[TMP21]], i64 10
-; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[TMP32]]
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <16 x float> [[TMP21]], i64 11
-; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[TMP33]]
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <16 x float> [[TMP21]], i64 12
-; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[TMP34]]
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <16 x float> [[TMP21]], i64 13
-; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP35]]
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <16 x float> [[TMP21]], i64 14
-; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
+; CHECK-NEXT: [[MP15:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 15
+; CHECK-NEXT: [[MV15:%.*]] = load float, ptr addrspace(4) [[MP15]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = fmul contract float [[IV15]], [[MV15]]
; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr addrspace(1) [[IP16]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr addrspace(4) [[MP16]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i64 0
+; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC26]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i64 1
+; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]]
+; CHECK-NEXT: [[IP18:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 27
+; CHECK-NEXT: [[MP18:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 18
+; CHECK-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr addrspace(1) [[IP18]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr addrspace(4) [[MP18]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <2 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x float> [[TMP13]], i64 0
+; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[TMP14]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP13]], i64 1
+; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[TMP15]]
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
-; CHECK-NEXT: [[TMP38:%.*]] = load <4 x float>, ptr addrspace(1) [[IP16]], align 4
-; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP20]], align 4
-; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> [[TMP39]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
-; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]]
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x float> [[TMP44]], i64 1
-; CHECK-NEXT: [[ACC28:%.*]] = fadd contract float [[ACC27]], [[TMP46]]
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x float> [[TMP44]], i64 2
-; CHECK-NEXT: [[ACC29:%.*]] = fadd contract float [[ACC28]], [[TMP47]]
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <8 x float> [[TMP44]], i64 3
-; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC29]], [[TMP48]]
-; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x float> [[TMP44]], i64 4
-; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC30]], [[TMP49]]
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x float> [[TMP44]], i64 5
-; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP50]]
-; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6
-; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]]
-; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7
-; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
-; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36
-; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4
-; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24
+; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 20
; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4
; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]]
-; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[PROD20]]
+; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC19]], [[PROD20]]
+; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33
+; CHECK-NEXT: [[MP21:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 21
+; CHECK-NEXT: [[TMP16:%.*]] = load <2 x float>, ptr addrspace(1) [[IP21]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = load <2 x float>, ptr addrspace(4) [[MP21]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = fmul contract <2 x float> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x float> [[TMP18]], i64 0
+; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP19]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x float> [[TMP18]], i64 1
+; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP20]]
+; CHECK-NEXT: [[IP23:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 35
+; CHECK-NEXT: [[MP23:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 23
+; CHECK-NEXT: [[TMP21:%.*]] = load <2 x float>, ptr addrspace(1) [[IP23]], align 4
+; CHECK-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr addrspace(4) [[MP23]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x float> [[TMP21]], [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[TMP23]], i64 0
+; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP24]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x float> [[TMP23]], i64 1
+; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[TMP25]]
; CHECK-NEXT: ret float [[ACC24]]
;
%ip0 = getelementptr inbounds float, ptr addrspace(1) %input, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
index a553fe126ee17..bad672c7743f6 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
@@ -9,10 +9,14 @@
define float @dot_contract(float %x) {
; CHECK-LABEL: @dot_contract(
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> <float 7.000000e+00, float 3.000000e+00, float 5.000000e+00, float 9.000000e+00>, [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call contract float @llvm.vector.reduce.fadd.v4f32(float [[X]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float 7.000000e+00, [[X:%.*]]
+; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[M0]], [[X]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float 3.000000e+00, [[X]]
+; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[M1]], [[A0]]
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float 5.000000e+00, [[X]]
+; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[M2]], [[A1]]
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float 9.000000e+00, [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract float [[M3]], [[A2]]
; CHECK-NEXT: ret float [[TMP4]]
;
%m0 = fmul contract float 7.000000e+00, %x
>From 414d1b00486e911084150dff43380a09b014c71c Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 21 Jul 2026 20:20:06 +0200
Subject: [PATCH 6/8] format
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 551d0fa919dce..f941c9462837a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30475,7 +30475,9 @@ class HorizontalReduction {
auto *FMul = dyn_cast<Instruction>(RdxVal);
if (FMul && FMul->getOpcode() == Instruction::FMul &&
FMul->hasOneUse() &&
- cast<FPMathOperator>(FMul)->getFastMathFlags().allowContract())
+ cast<FPMathOperator>(FMul)
+ ->getFastMathFlags()
+ .allowContract())
ReductionCost += FusionSaving;
}
}
>From 8bb79f9cc2f189cce65df7a7e509f58f10acb3e5 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 12 Aug 2026 01:04:58 +0200
Subject: [PATCH 7/8] adjust test
---
.../AMDGPU/ordered-reduction-fma-fusion.ll | 65 +++++++------------
1 file changed, 25 insertions(+), 40 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index 30174c1f7eb4e..ed4412708f848 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -9,27 +9,21 @@
define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-LABEL: @conv_contract(
; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
-; CHECK-NEXT: [[IV0:%.*]] = load float, ptr addrspace(1) [[IP0]], align 4
; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
-; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV0]], [[MV0]]
-; CHECK-NEXT: [[IP1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1
-; CHECK-NEXT: [[IV1:%.*]] = load float, ptr addrspace(1) [[IP1]], align 4
-; CHECK-NEXT: [[MP1:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1
-; CHECK-NEXT: [[MV1:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4
-; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[IV1]], [[MV1]]
+; CHECK-NEXT: [[TMP26:%.*]] = load <2 x float>, ptr addrspace(1) [[IP0]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = load <2 x float>, ptr addrspace(4) [[MP0]], align 4
+; CHECK-NEXT: [[TMP28:%.*]] = fmul contract <2 x float> [[TMP26]], [[TMP27]]
+; CHECK-NEXT: [[PROD0:%.*]] = extractelement <2 x float> [[TMP28]], i64 0
+; CHECK-NEXT: [[PROD1:%.*]] = extractelement <2 x float> [[TMP28]], i64 1
; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[PROD1]]
; CHECK-NEXT: [[IP2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 2
-; CHECK-NEXT: [[IV2:%.*]] = load float, ptr addrspace(1) [[IP2]], align 4
; CHECK-NEXT: [[MP2:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 2
-; CHECK-NEXT: [[MV2:%.*]] = load float, ptr addrspace(4) [[MP2]], align 4
-; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[IV2]], [[MV2]]
+; CHECK-NEXT: [[TMP29:%.*]] = load <2 x float>, ptr addrspace(1) [[IP2]], align 4
+; CHECK-NEXT: [[TMP30:%.*]] = load <2 x float>, ptr addrspace(4) [[MP2]], align 4
+; CHECK-NEXT: [[TMP31:%.*]] = fmul contract <2 x float> [[TMP29]], [[TMP30]]
+; CHECK-NEXT: [[PROD2:%.*]] = extractelement <2 x float> [[TMP31]], i64 0
; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
-; CHECK-NEXT: [[IP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 3
-; CHECK-NEXT: [[IV3:%.*]] = load float, ptr addrspace(1) [[IP3]], align 4
-; CHECK-NEXT: [[MP3:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 3
-; CHECK-NEXT: [[MV3:%.*]] = load float, ptr addrspace(4) [[MP3]], align 4
-; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[IV3]], [[MV3]]
+; CHECK-NEXT: [[PROD3:%.*]] = extractelement <2 x float> [[TMP31]], i64 1
; CHECK-NEXT: [[ACC3:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
; CHECK-NEXT: [[IV4:%.*]] = load float, ptr addrspace(1) [[IP4]], align 4
@@ -44,40 +38,31 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[PROD5:%.*]] = fmul contract float [[IV5]], [[MV5]]
; CHECK-NEXT: [[ACC5:%.*]] = fadd contract float [[ACC4]], [[PROD5]]
; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
-; CHECK-NEXT: [[IV6:%.*]] = load float, ptr addrspace(1) [[IP6]], align 4
; CHECK-NEXT: [[MP6:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 6
-; CHECK-NEXT: [[MV6:%.*]] = load float, ptr addrspace(4) [[MP6]], align 4
-; CHECK-NEXT: [[PROD6:%.*]] = fmul contract float [[IV6]], [[MV6]]
+; CHECK-NEXT: [[TMP32:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[TMP33:%.*]] = load <2 x float>, ptr addrspace(4) [[MP6]], align 4
+; CHECK-NEXT: [[TMP34:%.*]] = fmul contract <2 x float> [[TMP32]], [[TMP33]]
+; CHECK-NEXT: [[PROD6:%.*]] = extractelement <2 x float> [[TMP34]], i64 0
; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC5]], [[PROD6]]
-; CHECK-NEXT: [[IP7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 10
-; CHECK-NEXT: [[IV7:%.*]] = load float, ptr addrspace(1) [[IP7]], align 4
-; CHECK-NEXT: [[MP7:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 7
-; CHECK-NEXT: [[MV7:%.*]] = load float, ptr addrspace(4) [[MP7]], align 4
-; CHECK-NEXT: [[PROD7:%.*]] = fmul contract float [[IV7]], [[MV7]]
+; CHECK-NEXT: [[PROD7:%.*]] = extractelement <2 x float> [[TMP34]], i64 1
; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[PROD7]]
; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
-; CHECK-NEXT: [[IV8:%.*]] = load float, ptr addrspace(1) [[IP8]], align 4
; CHECK-NEXT: [[MP8:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 8
-; CHECK-NEXT: [[MV8:%.*]] = load float, ptr addrspace(4) [[MP8]], align 4
-; CHECK-NEXT: [[PROD8:%.*]] = fmul contract float [[IV8]], [[MV8]]
+; CHECK-NEXT: [[TMP35:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
+; CHECK-NEXT: [[TMP36:%.*]] = load <2 x float>, ptr addrspace(4) [[MP8]], align 4
+; CHECK-NEXT: [[TMP38:%.*]] = fmul contract <2 x float> [[TMP35]], [[TMP36]]
+; CHECK-NEXT: [[PROD8:%.*]] = extractelement <2 x float> [[TMP38]], i64 0
; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[PROD8]]
-; CHECK-NEXT: [[IP9:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12
-; CHECK-NEXT: [[IV9:%.*]] = load float, ptr addrspace(1) [[IP9]], align 4
-; CHECK-NEXT: [[MP9:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 9
-; CHECK-NEXT: [[MV9:%.*]] = load float, ptr addrspace(4) [[MP9]], align 4
-; CHECK-NEXT: [[PROD9:%.*]] = fmul contract float [[IV9]], [[MV9]]
+; CHECK-NEXT: [[PROD9:%.*]] = extractelement <2 x float> [[TMP38]], i64 1
; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[PROD9]]
; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
-; CHECK-NEXT: [[IV10:%.*]] = load float, ptr addrspace(1) [[IP10]], align 4
; CHECK-NEXT: [[MP10:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 10
-; CHECK-NEXT: [[MV10:%.*]] = load float, ptr addrspace(4) [[MP10]], align 4
-; CHECK-NEXT: [[PROD10:%.*]] = fmul contract float [[IV10]], [[MV10]]
+; CHECK-NEXT: [[TMP39:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
+; CHECK-NEXT: [[TMP40:%.*]] = load <2 x float>, ptr addrspace(4) [[MP10]], align 4
+; CHECK-NEXT: [[TMP41:%.*]] = fmul contract <2 x float> [[TMP39]], [[TMP40]]
+; CHECK-NEXT: [[PROD10:%.*]] = extractelement <2 x float> [[TMP41]], i64 0
; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[PROD10]]
-; CHECK-NEXT: [[IP11:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17
-; CHECK-NEXT: [[IV11:%.*]] = load float, ptr addrspace(1) [[IP11]], align 4
-; CHECK-NEXT: [[MP11:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 11
-; CHECK-NEXT: [[MV11:%.*]] = load float, ptr addrspace(4) [[MP11]], align 4
-; CHECK-NEXT: [[PROD11:%.*]] = fmul contract float [[IV11]], [[MV11]]
+; CHECK-NEXT: [[PROD11:%.*]] = extractelement <2 x float> [[TMP41]], i64 1
; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[PROD11]]
; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
; CHECK-NEXT: [[IV12:%.*]] = load float, ptr addrspace(1) [[IP12]], align 4
>From 3b29f23e914864e5f4979226dbc3148f80b0c616 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 14 Aug 2026 16:21:16 +0200
Subject: [PATCH 8/8] re-apply the comment
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 36 ++--
.../AMDGPU/ordered-reduction-fma-fusion.ll | 177 ++++++++----------
2 files changed, 99 insertions(+), 114 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 44d49a85e50f2..3e082dffac804 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -31116,22 +31116,28 @@ class HorizontalReduction {
RdxFMF.allowContract()) {
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *Ty = VL.front()->getType();
- IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty},
- RdxFMF);
- InstructionCost FusionSaving =
+ InstructionCost UnfusedCost =
TTI->getArithmeticInstrCost(Instruction::FMul, Ty, CostKind) +
- TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind) -
- TTI->getIntrinsicInstrCost(ICA, CostKind);
- if (FusionSaving.isValid() && FusionSaving > 0)
- for (Value *RdxVal : VL) {
- auto *FMul = dyn_cast<Instruction>(RdxVal);
- if (FMul && FMul->getOpcode() == Instruction::FMul &&
- FMul->hasOneUse() &&
- cast<FPMathOperator>(FMul)
- ->getFastMathFlags()
- .allowContract())
- ReductionCost += FusionSaving;
- }
+ TTI->getArithmeticInstrCost(Instruction::FAdd, Ty, CostKind);
+ for (Value *RdxVal : VL) {
+ auto *FMul = dyn_cast<Instruction>(RdxVal);
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse())
+ continue;
+ auto *FAdd = dyn_cast<Instruction>(FMul->user_back());
+ if (!FAdd)
+ continue;
+ InstructionsState FAddS(FAdd, FAdd);
+ if (!FAddS.isAddSubLikeOp())
+ continue;
+ InstructionCost FMACost =
+ canConvertToFMA(FAdd, FAddS, DT, DL, *TTI, TLI);
+ if (!FMACost.isValid())
+ continue;
+ InstructionCost FusionSaving = UnfusedCost - FMACost;
+ if (FusionSaving.isValid() && FusionSaving > 0)
+ ReductionCost += FusionSaving;
+ }
}
} else {
ReductionCost =
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index ed4412708f848..4574ff3f578bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -10,123 +10,102 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-LABEL: @conv_contract(
; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = load <2 x float>, ptr addrspace(1) [[IP0]], align 4
-; CHECK-NEXT: [[TMP27:%.*]] = load <2 x float>, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[TMP28:%.*]] = fmul contract <2 x float> [[TMP26]], [[TMP27]]
-; CHECK-NEXT: [[PROD0:%.*]] = extractelement <2 x float> [[TMP28]], i64 0
-; CHECK-NEXT: [[PROD1:%.*]] = extractelement <2 x float> [[TMP28]], i64 1
-; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[PROD1]]
-; CHECK-NEXT: [[IP2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 2
-; CHECK-NEXT: [[MP2:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 2
-; CHECK-NEXT: [[TMP29:%.*]] = load <2 x float>, ptr addrspace(1) [[IP2]], align 4
-; CHECK-NEXT: [[TMP30:%.*]] = load <2 x float>, ptr addrspace(4) [[MP2]], align 4
-; CHECK-NEXT: [[TMP31:%.*]] = fmul contract <2 x float> [[TMP29]], [[TMP30]]
-; CHECK-NEXT: [[PROD2:%.*]] = extractelement <2 x float> [[TMP31]], i64 0
-; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
-; CHECK-NEXT: [[PROD3:%.*]] = extractelement <2 x float> [[TMP31]], i64 1
-; CHECK-NEXT: [[ACC3:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
; CHECK-NEXT: [[IV4:%.*]] = load float, ptr addrspace(1) [[IP4]], align 4
-; CHECK-NEXT: [[MP4:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 4
-; CHECK-NEXT: [[MV4:%.*]] = load float, ptr addrspace(4) [[MP4]], align 4
-; CHECK-NEXT: [[PROD4:%.*]] = fmul contract float [[IV4]], [[MV4]]
-; CHECK-NEXT: [[ACC4:%.*]] = fadd contract float [[ACC3]], [[PROD4]]
; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4
-; CHECK-NEXT: [[MP5:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 5
-; CHECK-NEXT: [[MV5:%.*]] = load float, ptr addrspace(4) [[MP5]], align 4
-; CHECK-NEXT: [[PROD5:%.*]] = fmul contract float [[IV5]], [[MV5]]
-; CHECK-NEXT: [[ACC5:%.*]] = fadd contract float [[ACC4]], [[PROD5]]
; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
-; CHECK-NEXT: [[MP6:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 6
-; CHECK-NEXT: [[TMP32:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
-; CHECK-NEXT: [[TMP33:%.*]] = load <2 x float>, ptr addrspace(4) [[MP6]], align 4
-; CHECK-NEXT: [[TMP34:%.*]] = fmul contract <2 x float> [[TMP32]], [[TMP33]]
-; CHECK-NEXT: [[PROD6:%.*]] = extractelement <2 x float> [[TMP34]], i64 0
-; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC5]], [[PROD6]]
-; CHECK-NEXT: [[PROD7:%.*]] = extractelement <2 x float> [[TMP34]], i64 1
-; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[PROD7]]
; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
-; CHECK-NEXT: [[MP8:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 8
-; CHECK-NEXT: [[TMP35:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
-; CHECK-NEXT: [[TMP36:%.*]] = load <2 x float>, ptr addrspace(4) [[MP8]], align 4
-; CHECK-NEXT: [[TMP38:%.*]] = fmul contract <2 x float> [[TMP35]], [[TMP36]]
-; CHECK-NEXT: [[PROD8:%.*]] = extractelement <2 x float> [[TMP38]], i64 0
-; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[PROD8]]
-; CHECK-NEXT: [[PROD9:%.*]] = extractelement <2 x float> [[TMP38]], i64 1
-; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[PROD9]]
; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
-; CHECK-NEXT: [[MP10:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 10
-; CHECK-NEXT: [[TMP39:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
-; CHECK-NEXT: [[TMP40:%.*]] = load <2 x float>, ptr addrspace(4) [[MP10]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = fmul contract <2 x float> [[TMP39]], [[TMP40]]
-; CHECK-NEXT: [[PROD10:%.*]] = extractelement <2 x float> [[TMP41]], i64 0
-; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[PROD10]]
-; CHECK-NEXT: [[PROD11:%.*]] = extractelement <2 x float> [[TMP41]], i64 1
-; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[PROD11]]
; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
-; CHECK-NEXT: [[IV12:%.*]] = load float, ptr addrspace(1) [[IP12]], align 4
-; CHECK-NEXT: [[MP12:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 12
-; CHECK-NEXT: [[MV12:%.*]] = load float, ptr addrspace(4) [[MP12]], align 4
-; CHECK-NEXT: [[PROD12:%.*]] = fmul contract float [[IV12]], [[MV12]]
-; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[PROD12]]
-; CHECK-NEXT: [[IP13:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19
-; CHECK-NEXT: [[MP13:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 13
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[IP13]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(4) [[MP13]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
-; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
-; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP5]]
+; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20
+; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4
; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4
-; CHECK-NEXT: [[MP15:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 15
-; CHECK-NEXT: [[MV15:%.*]] = load float, ptr addrspace(4) [[MP15]], align 4
-; CHECK-NEXT: [[TMP37:%.*]] = fmul contract float [[IV15]], [[MV15]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV4]], i64 4
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV5]], i64 5
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <16 x float> [[TMP8]], float [[IV14]], i64 14
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[IV15]], i64 15
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP24]], <16 x float> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP20]], [[TMP6]]
+; CHECK-NEXT: [[ACC25:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
-; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
-; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
-; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr addrspace(1) [[IP16]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr addrspace(4) [[MP16]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC26]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]]
-; CHECK-NEXT: [[IP18:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 27
-; CHECK-NEXT: [[MP18:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 18
-; CHECK-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr addrspace(1) [[IP18]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr addrspace(4) [[MP18]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <2 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x float> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP21]], i64 4
; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[TMP14]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP13]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP21]], i64 5
; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[TMP15]]
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x float> [[TMP21]], i64 6
+; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC19]], [[TMP28]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <16 x float> [[TMP21]], i64 7
+; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[TMP29]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <16 x float> [[TMP21]], i64 8
+; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[TMP30]]
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <16 x float> [[TMP21]], i64 9
+; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[TMP31]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <16 x float> [[TMP21]], i64 10
+; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[TMP32]]
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <16 x float> [[TMP21]], i64 11
+; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[TMP33]]
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <16 x float> [[TMP21]], i64 12
+; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[TMP34]]
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <16 x float> [[TMP21]], i64 13
+; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP35]]
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <16 x float> [[TMP21]], i64 14
+; CHECK-NEXT: [[ACC14:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
+; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[TMP53]]
+; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
+; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
-; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4
-; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 20
+; CHECK-NEXT: [[TMP38:%.*]] = load <4 x float>, ptr addrspace(1) [[IP16]], align 4
+; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4
+; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> [[TMP39]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
+; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC15]], [[TMP45]]
+; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x float> [[TMP44]], i64 1
+; CHECK-NEXT: [[ACC28:%.*]] = fadd contract float [[ACC27]], [[TMP46]]
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x float> [[TMP44]], i64 2
+; CHECK-NEXT: [[ACC29:%.*]] = fadd contract float [[ACC28]], [[TMP47]]
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <8 x float> [[TMP44]], i64 3
+; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC29]], [[TMP48]]
+; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x float> [[TMP44]], i64 4
+; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC30]], [[TMP49]]
+; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x float> [[TMP44]], i64 5
+; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP50]]
+; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6
+; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]]
+; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7
+; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
+; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36
+; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4
+; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24
; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4
; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]]
-; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC19]], [[PROD20]]
-; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33
-; CHECK-NEXT: [[MP21:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 21
-; CHECK-NEXT: [[TMP16:%.*]] = load <2 x float>, ptr addrspace(1) [[IP21]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = load <2 x float>, ptr addrspace(4) [[MP21]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = fmul contract <2 x float> [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x float> [[TMP18]], i64 0
-; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP19]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x float> [[TMP18]], i64 1
-; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP20]]
-; CHECK-NEXT: [[IP23:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 35
-; CHECK-NEXT: [[MP23:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 23
-; CHECK-NEXT: [[TMP21:%.*]] = load <2 x float>, ptr addrspace(1) [[IP23]], align 4
-; CHECK-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr addrspace(4) [[MP23]], align 4
-; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x float> [[TMP21]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[TMP23]], i64 0
-; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP24]]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x float> [[TMP23]], i64 1
-; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[TMP25]]
+; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[PROD20]]
; CHECK-NEXT: ret float [[ACC24]]
;
%ip0 = getelementptr inbounds float, ptr addrspace(1) %input, i64 0
More information about the llvm-commits
mailing list