[llvm-branch-commits] [llvm] [SLP] Cancel the phantom load saving on fadd reductions that lose an fma (PR #228903)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 4 08:12:54 PDT 2026
https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/228903
Add TargetTransformInfo::consecutiveLoadsCoalesce, implemented by
AMDGPU, whose consecutive scalar loads already coalesce into one wide
access. On such a target cancel the load saving of a contract fadd
reduction over contract fmuls, which otherwise trades every scalar fma
for a saving that never materializes. A reassociable reduction keeps it,
as its vector fmuls fuse into the reduction.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 66600068db486b09d8cb8e50ef6f6fa032de857f Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 27 Sep 2026 22:14:39 +0200
Subject: [PATCH] [SLP] Cancel the phantom load saving on fadd reductions that
lose an fma
Add TargetTransformInfo::consecutiveLoadsCoalesce, implemented by
AMDGPU, whose consecutive scalar loads already coalesce into one wide
access. On such a target cancel the load saving of a contract fadd
reduction over contract fmuls, which otherwise trades every scalar fma
for a saving that never materializes. A reassociable reduction keeps it,
as its vector fmuls fuse into the reduction.
---
.../llvm/Analysis/TargetTransformInfo.h | 7 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 +
llvm/lib/Analysis/TargetTransformInfo.cpp | 7 +
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 22 +
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 2 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 106 +++-
.../CodeGen/AMDGPU/slp-coalesced-loads.ll | 233 ++++----
.../AMDGPU/elementwise-coalesced-loads.ll | 180 +++++--
.../ordered-reduction-coalesced-loads.ll | 508 +++++++++++++-----
.../AMDGPU/ordered-reduction-fma-fusion.ll | 194 ++++---
10 files changed, 878 insertions(+), 387 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index a33e6f62e941ed..9613a1679b7f96 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1946,6 +1946,13 @@ class TargetTransformInfo {
/// load/store in the given address space.
LLVM_ABI unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const;
+ /// \returns True if the backend coalesces \p NumElts consecutive scalar
+ /// loads of \p ElemTy in the given address space into one wider access,
+ /// \p Alignment being the best alignment known among the loads.
+ LLVM_ABI bool consecutiveLoadsCoalesce(Type *ElemTy, unsigned NumElts,
+ Align Alignment,
+ unsigned AddrSpace) const;
+
/// \returns True if the load instruction is legal to vectorize.
LLVM_ABI bool isLegalToVectorizeLoad(LoadInst *LI) const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a19c122c16f20c..418c806cc37289 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1129,6 +1129,12 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 128;
}
+ virtual bool consecutiveLoadsCoalesce(Type *ElemTy, unsigned NumElts,
+ Align Alignment,
+ unsigned AddrSpace) const {
+ return false;
+ }
+
virtual bool isLegalToVectorizeLoad(LoadInst *LI) const { return true; }
virtual bool isLegalToVectorizeStore(StoreInst *SI) const { return true; }
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index af73a615f25fa6..99ac456ee2eb05 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1443,6 +1443,13 @@ unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
return TTIImpl->getLoadStoreVecRegBitWidth(AS);
}
+bool TargetTransformInfo::consecutiveLoadsCoalesce(Type *ElemTy,
+ unsigned NumElts,
+ Align Alignment,
+ unsigned AS) const {
+ return TTIImpl->consecutiveLoadsCoalesce(ElemTy, NumElts, Alignment, AS);
+}
+
bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
return TTIImpl->isLegalToVectorizeLoad(LI);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 9662be4530f785..70ea18254700d1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -390,6 +390,28 @@ unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
return 128;
}
+bool GCNTTIImpl::consecutiveLoadsCoalesce(Type *ElemTy, unsigned NumElts,
+ Align Alignment,
+ unsigned AddrSpace) const {
+ unsigned MaxBits = getLoadStoreVecRegBitWidth(AddrSpace);
+ unsigned ElemBits = DL.getTypeSizeInBits(ElemTy);
+ if (MaxBits < 64 || ElemBits % 8 != 0)
+ return false;
+ unsigned Bits = std::min(ElemBits * NumElts, MaxBits);
+ if (!isLegalToVectorizeLoadChain(Bits / 8, Alignment, AddrSpace))
+ return false;
+ if (Alignment.value() % (Bits / 8) == 0)
+ return true;
+ LLVMContext &Ctx = ElemTy->getContext();
+ unsigned VecSpeed = 0, ElemSpeed = 0;
+ if (!allowsMisalignedMemoryAccesses(Ctx, Bits, AddrSpace, Alignment,
+ &VecSpeed))
+ return false;
+ allowsMisalignedMemoryAccesses(Ctx, ElemBits, AddrSpace, Alignment,
+ &ElemSpeed);
+ return VecSpeed >= ElemSpeed;
+}
+
bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
Align Alignment,
unsigned AddrSpace) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 593aa5ebc43a26..8bdd6d45683836 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -140,6 +140,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
unsigned ChainSizeInBytes,
VectorType *VecTy) const override;
unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override;
+ bool consecutiveLoadsCoalesce(Type *ElemTy, unsigned NumElts, Align Alignment,
+ unsigned AddrSpace) const override;
bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
unsigned AddrSpace) const;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 25953e843be405..26a61193083608 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -473,11 +473,23 @@ class slpvectorizer::BoUpSLP {
TargetTransformInfo::TargetCostKind getCostKind() const { return CostKind; }
+ /// Which vectorized load bundles lose the load saving that never
+ /// materializes on targets whose consecutive scalar loads coalesce into the
+ /// same wide access.
+ enum class CoalescedLoadSavings {
+ /// Every bundle keeps its saving.
+ Keep,
+ /// Every clean bundle loses its saving.
+ Cancel,
+ };
+
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
- /// final cost.
+ /// final cost. \p LoadSavings selects the load bundles whose saving is
+ /// cancelled.
InstructionCost
- calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals = {},
- Instruction *RdxRoot = nullptr);
+ calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
+ Instruction *RdxRoot,
+ CoalescedLoadSavings LoadSavings);
/// Finds i1 and/or nodes whose operand nodes are booleanized wide leaves
/// (one-use truncs of wide values or zero-tests of values from [0, 1]) and
@@ -2479,6 +2491,15 @@ class slpvectorizer::BoUpSLP {
const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
bool &IsProfitableToDemote, bool IsTruncRoot) const;
+ /// \returns the load saving credited to the vectorized load bundle \p TE
+ /// that never materializes on a target whose consecutive scalar loads
+ /// coalesce into the same wide access, or zero if \p LoadSavings keeps it.
+ /// Only clean bundles with no reordering, reuse or bit-width reduction lose
+ /// the saving.
+ InstructionCost
+ getCoalescedLoadPhantomSaving(const TreeEntry &TE,
+ CoalescedLoadSavings LoadSavings) const;
+
/// Builds the list of reorderable operands on the edges \p Edges of the \p
/// UserTE, which allow reordering (i.e. the operands can be reordered because
/// they have only one user and reordarable).
@@ -13408,6 +13429,23 @@ InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const {
return Penalty;
}
+/// \returns which load bundles of a reduction over \p VL lose the saving they
+/// are credited for although their scalar loads coalesce anyway. A reduction
+/// that would lose an fma has a stake in every load bundle, and any other
+/// reduction keeps every saving. A reassociable reduction loses no fma because
+/// its vector fmuls fuse into the reduction.
+static BoUpSLP::CoalescedLoadSavings
+getReductionCoalescedLoadSavings(RecurKind RdxKind, FastMathFlags RdxFMF,
+ ArrayRef<Value *> VL) {
+ if (RdxKind != RecurKind::FAdd || !RdxFMF.allowContract() ||
+ RdxFMF.allowReassoc() || none_of(VL, [](Value *V) {
+ return match(V,
+ m_OneUse(m_AllowContract(m_FMul(m_Value(), m_Value()))));
+ }))
+ return BoUpSLP::CoalescedLoadSavings::Keep;
+ return BoUpSLP::CoalescedLoadSavings::Cancel;
+}
+
// A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
// are already folded into TreeCost like any other entry, including all
// shuffle/insert/extract overhead elsewhere in the tree. So bypassing the
@@ -18556,6 +18594,43 @@ bool BoUpSLP::isTreeNotExtendable() const {
return Res;
}
+InstructionCost
+BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE,
+ CoalescedLoadSavings LoadSavings) const {
+ if (LoadSavings == CoalescedLoadSavings::Keep)
+ return 0;
+ if (!TE.hasState() || TE.isGather() || TE.getOpcode() != Instruction::Load ||
+ TE.State != TreeEntry::Vectorize || TE.getInterleaveFactor())
+ return 0;
+ if (DeletedNodes.contains(&TE) || TransformedToGatherNodes.contains(&TE))
+ return 0;
+ if (!TE.ReuseShuffleIndices.empty() || !TE.ReorderIndices.empty() ||
+ MinBWs.contains(&TE))
+ return 0;
+ if (!all_of(TE.Scalars, IsaPred<LoadInst>))
+ return 0;
+ auto *LI0 = cast<LoadInst>(TE.getMainOp());
+ Align BestAlign = LI0->getAlign();
+ for (Value *V : TE.Scalars)
+ BestAlign = std::max(BestAlign, cast<LoadInst>(V)->getAlign());
+ if (!TTI->consecutiveLoadsCoalesce(LI0->getType(), TE.Scalars.size(),
+ BestAlign, LI0->getPointerAddressSpace()))
+ return 0;
+ InstructionCost ScalarLdCost = 0;
+ for (Value *V : TE.Scalars) {
+ auto *LI = cast<LoadInst>(V);
+ ScalarLdCost +=
+ TTI->getMemoryOpCost(Instruction::Load, LI->getType(), LI->getAlign(),
+ LI->getPointerAddressSpace(), CostKind,
+ TTI::getOperandInfo(LI->getPointerOperand()), LI);
+ }
+ Type *VecTy = getWidenedType(LI0->getType(), TE.Scalars.size());
+ InstructionCost VecLdCost = TTI->getMemoryOpCost(
+ Instruction::Load, VecTy, LI0->getAlign(), LI0->getPointerAddressSpace(),
+ CostKind, TTI::getOperandInfo(LI0->getPointerOperand()));
+ return ScalarLdCost - VecLdCost;
+}
+
InstructionCost BoUpSLP::getSpillCost() {
// Walk the vectorizable tree from the root towards its leaves, tracking
// which vectorized operand values would be live across each tree edge
@@ -19322,9 +19397,9 @@ void BoUpSLP::detectBooleanizedNodes() {
}
}
-InstructionCost
-BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
- Instruction *RdxRoot) {
+InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
+ ArrayRef<Value *> VectorizedVals, Instruction *RdxRoot,
+ CoalescedLoadSavings LoadSavings) {
// FIXME: support buildvector of the gather nodes with struct types.
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() &&
@@ -19448,6 +19523,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
"Expected gather nodes with users only.");
InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts);
+ C += getCoalescedLoadPhantomSaving(TE, LoadSavings);
uint64_t Scale = 0;
bool CostIsFree = C == 0;
// For gather/buildvector (and split-vectorize) entries, prefer the
@@ -19650,6 +19726,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
};
auto RecostEntry = [&](const TreeEntry *TE) {
InstructionCost C = getEntryCost(TE, VectorizedVals, CheckedExtracts);
+ C += getCoalescedLoadPhantomSaving(*TE, LoadSavings);
if (!C.isValid() || C == 0)
return C;
uint64_t Scale = EntryToScale.lookup(TE);
@@ -29640,7 +29717,9 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
R.transformNodes();
R.computeMinimumValueSizes();
- InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+ InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable(
+ /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr,
+ BoUpSLP::CoalescedLoadSavings::Keep);
R.buildExternalUses();
Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries();
@@ -30722,7 +30801,9 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
}
R.transformNodes();
R.computeMinimumValueSizes();
- InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
+ InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable(
+ /*VectorizedVals=*/{}, /*RdxRoot=*/nullptr,
+ BoUpSLP::CoalescedLoadSavings::Keep);
R.buildExternalUses();
InstructionCost Cost = R.getTreeCost(TreeCost);
@@ -33025,7 +33106,9 @@ class HorizontalReduction {
}
V.transformNodes();
V.computeMinimumValueSizes();
- InstructionCost TreeCost = V.calculateTreeCostAndTrimNonProfitable(VL);
+ InstructionCost TreeCost = V.calculateTreeCostAndTrimNonProfitable(
+ VL, /*RdxRoot=*/nullptr,
+ getReductionCoalescedLoadSavings(RdxKind, RdxFMF, VL));
// A negated slice is subtracted in the final combine, it cannot be
// accumulated lane-wise.
const bool LoopAccCandidate =
@@ -33679,8 +33762,9 @@ class HorizontalReduction {
V.transformNodes();
V.computeMinimumValueSizes();
- InstructionCost TreeCost =
- V.calculateTreeCostAndTrimNonProfitable(VL, RdxRootInst);
+ InstructionCost TreeCost = V.calculateTreeCostAndTrimNonProfitable(
+ VL, RdxRootInst,
+ getReductionCoalescedLoadSavings(RdxKind, RdxFMF, VL));
V.buildExternalUses(LocalExternallyUsedValues);
InstructionCost VectorCost, RdxOpCost;
diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
index 4db68adac00aad..281221e8181fa6 100644
--- a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
@@ -7,24 +7,20 @@ define float @dot8_ordered(ptr addrspace(1) noalias readonly align 4 %a, ptr add
; GFX90A-LABEL: dot8_ordered:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
-; GFX90A-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:16
-; GFX90A-NEXT: global_load_dwordx4 v[14:17], v[2:3], off offset:16
-; GFX90A-NEXT: global_load_dwordx4 v[18:21], v[2:3], off
-; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
+; GFX90A-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
+; GFX90A-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[18:21], v[2:3], off offset:16
+; GFX90A-NEXT: s_waitcnt vmcnt(2)
+; GFX90A-NEXT: v_fma_f32 v0, v10, v6, v4
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v11, v7
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v12, v8
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v13, v9
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[18:19]
-; GFX90A-NEXT: v_add_f32_e32 v4, v4, v6
-; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[8:9], v[20:21]
-; GFX90A-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[10:11], v[14:15]
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v8
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v9
-; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v15, v19
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v16, v20
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v17, v21
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot8_ordered:
@@ -58,24 +54,19 @@ define float @dot8_ordered(ptr addrspace(1) noalias readonly align 4 %a, ptr add
; GFX1250-NEXT: global_load_b128 v[18:21], v[2:3], off offset:16
; GFX1250-NEXT: s_wait_loadcnt 0x2
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[6:7]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[12:13], v[8:9]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v4, v0
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: v_fma_f32 v0, v10, v6, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v11, v7
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v12, v8
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v13, v9
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[14:15], v[18:19]
-; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[16:17], v[20:21]
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v14, v18
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v15, v19
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v16, v20
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v17, v21
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
entry:
%pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
@@ -246,17 +237,15 @@ define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %ac
; GFX90A-NEXT: global_load_dwordx2 v[10:11], v[0:1], off
; GFX90A-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX90A-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
-; GFX90A-NEXT: v_add_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[8:9]
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_fma_f32 v0, v0, v6, v4
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v1, v7
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v2, v8
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v3, v9
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot_i16_window:
@@ -1099,24 +1088,20 @@ define float @dot8_ordered_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b, flo
; GFX90A-LABEL: dot8_ordered_lds_a16:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b128 v[4:7], v0
-; GFX90A-NEXT: ds_read_b128 v[8:11], v0 offset:16
-; GFX90A-NEXT: ds_read_b128 v[12:15], v1 offset:16
-; GFX90A-NEXT: ds_read_b128 v[16:19], v1
-; GFX90A-NEXT: s_waitcnt lgkmcnt(1)
-; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[12:13]
+; GFX90A-NEXT: ds_read_b128 v[4:7], v1
+; GFX90A-NEXT: ds_read_b128 v[8:11], v0
+; GFX90A-NEXT: ds_read_b128 v[12:15], v0 offset:16
+; GFX90A-NEXT: ds_read_b128 v[16:19], v1 offset:16
+; GFX90A-NEXT: s_waitcnt lgkmcnt(2)
+; GFX90A-NEXT: v_fma_f32 v0, v8, v4, v2
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v9, v5
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v11, v7
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[18:19]
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v5
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v7
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v8
-; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[14:15]
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v9
-; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v15, v19
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot8_ordered_lds_a16:
@@ -1147,23 +1132,19 @@ define float @dot8_ordered_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b, flo
; GFX1250-NEXT: ds_load_b128 v[12:15], v0 offset:16
; GFX1250-NEXT: ds_load_b128 v[16:19], v1 offset:16
; GFX1250-NEXT: s_wait_dscnt 0x2
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[8:9], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[10:11], v[6:7]
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: v_fma_f32 v0, v8, v4, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v9, v5
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v11, v7
; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
-; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[14:15], v[18:19]
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v12, v16
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v15, v19
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
entry:
%pa0 = getelementptr inbounds float, ptr addrspace(3) %a, i32 0
@@ -1221,23 +1202,20 @@ define float @dot8_ordered_noacc(ptr addrspace(1) noalias readonly align 4 %a, p
; GFX90A-LABEL: dot8_ordered_noacc:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX90A-NEXT: global_load_dwordx4 v[12:15], v[2:3], off offset:16
-; GFX90A-NEXT: global_load_dwordx4 v[16:19], v[2:3], off
-; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[10:11], v[14:15]
+; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[2:3], off
+; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16
+; GFX90A-NEXT: global_load_dwordx4 v[16:19], v[2:3], off offset:16
+; GFX90A-NEXT: s_waitcnt vmcnt(2)
+; GFX90A-NEXT: v_mul_f32_e32 v0, v9, v5
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v11, v7
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
-; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[6:7], v[18:19]
-; GFX90A-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_pk_mul_f32 v[6:7], v[8:9], v[12:13]
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v7
-; GFX90A-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v14, v18
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v15, v19
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot8_ordered_noacc:
@@ -1271,23 +1249,19 @@ define float @dot8_ordered_noacc(ptr addrspace(1) noalias readonly align 4 %a, p
; GFX1250-NEXT: global_load_b128 v[16:19], v[2:3], off offset:16
; GFX1250-NEXT: s_wait_loadcnt 0x2
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[8:9], v[4:5]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[10:11], v[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v2, v0, v2
+; GFX1250-NEXT: v_mul_f32_e32 v0, v9, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v10, v6
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v11, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[12:13], v[16:17]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: v_pk_mul_f32 v[2:3], v[14:15], v[18:19]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v12, v16
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v13, v17
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v14, v18
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v15, v19
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
entry:
%pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
@@ -1347,34 +1321,31 @@ define float @dot8_sitofp_i16(ptr addrspace(1) noalias readonly align 16 %a, ptr
; GFX90A-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
; GFX90A-NEXT: global_load_dwordx4 v[8:11], v[2:3], off
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v13, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v12, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v14, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v17, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v16, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v19, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v18, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX90A-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[14:15]
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX90A-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[16:17]
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX90A-NEXT: v_pk_mul_f32 v[8:9], v[12:13], v[18:19]
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v8
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v9
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v3, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v2, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v8, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v9, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v12, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v13, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v10, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v1, v2
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v14, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v15, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v4, v8
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v5, v9
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v1, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v12, v13
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v6, v10
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v14, v15
+; GFX90A-NEXT: v_fmac_f32_e32 v0, v7, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot8_sitofp_i16:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
index 1948fe41f9af8a..67584277cb976b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll
@@ -11,37 +11,155 @@
; the fadd chain. The gfx9 targets subtract the zero point from four lanes
; as a vector, gfx1030 keeps everything scalar.
define float @dequant8_ordered(ptr addrspace(1) %mat, ptr addrspace(1) %vec, half %zph, float %acc) {
-; CHECK-LABEL: define float @dequant8_ordered(
-; CHECK-SAME: ptr addrspace(1) [[MAT:%.*]], ptr addrspace(1) [[VEC:%.*]], half [[ZPH:%.*]], float [[ACC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[ZP:%.*]] = fpext half [[ZPH]] to float
-; CHECK-NEXT: br label %[[BODY:.*]]
-; CHECK: [[BODY]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x half>, ptr addrspace(1) [[VEC]], align 16
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr addrspace(1) [[MAT]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[TMP0]] to <8 x float>
-; CHECK-NEXT: [[TMP3:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x float> poison, float [[ZP]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fsub contract <8 x float> [[TMP3]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = fmul contract <8 x float> [[TMP6]], [[TMP2]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP7]], i64 0
-; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[ACC]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP7]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[A0]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP7]], i64 2
-; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[A1]], [[TMP10]]
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x float> [[TMP7]], i64 3
-; CHECK-NEXT: [[A3:%.*]] = fadd contract float [[A2]], [[TMP11]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[TMP7]], i64 4
-; CHECK-NEXT: [[A4:%.*]] = fadd contract float [[A3]], [[TMP12]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x float> [[TMP7]], i64 5
-; CHECK-NEXT: [[A5:%.*]] = fadd contract float [[A4]], [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x float> [[TMP7]], i64 6
-; CHECK-NEXT: [[A6:%.*]] = fadd contract float [[A5]], [[TMP14]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x float> [[TMP7]], i64 7
-; CHECK-NEXT: [[A7:%.*]] = fadd contract float [[A6]], [[TMP15]]
-; CHECK-NEXT: ret float [[A7]]
+; GFX9-LABEL: define float @dequant8_ordered(
+; GFX9-SAME: ptr addrspace(1) [[MAT:%.*]], ptr addrspace(1) [[VEC:%.*]], half [[ZPH:%.*]], float [[ACC:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[ZP:%.*]] = fpext half [[ZPH]] to float
+; GFX9-NEXT: br label %[[BODY:.*]]
+; GFX9: [[BODY]]:
+; GFX9-NEXT: [[V0:%.*]] = load half, ptr addrspace(1) [[VEC]], align 16
+; GFX9-NEXT: [[VP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 2
+; GFX9-NEXT: [[V1:%.*]] = load half, ptr addrspace(1) [[VP1]], align 2
+; GFX9-NEXT: [[VP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 4
+; GFX9-NEXT: [[V2:%.*]] = load half, ptr addrspace(1) [[VP2]], align 4
+; GFX9-NEXT: [[VP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 6
+; GFX9-NEXT: [[V3:%.*]] = load half, ptr addrspace(1) [[VP3]], align 2
+; GFX9-NEXT: [[VP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 8
+; GFX9-NEXT: [[V4:%.*]] = load half, ptr addrspace(1) [[VP4]], align 8
+; GFX9-NEXT: [[VP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 10
+; GFX9-NEXT: [[V5:%.*]] = load half, ptr addrspace(1) [[VP5]], align 2
+; GFX9-NEXT: [[VP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 12
+; GFX9-NEXT: [[V6:%.*]] = load half, ptr addrspace(1) [[VP6]], align 4
+; GFX9-NEXT: [[VP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 14
+; GFX9-NEXT: [[V7:%.*]] = load half, ptr addrspace(1) [[VP7]], align 2
+; GFX9-NEXT: [[MP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 4
+; GFX9-NEXT: [[Q4:%.*]] = load i8, ptr addrspace(1) [[MP4]], align 2
+; GFX9-NEXT: [[MP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 5
+; GFX9-NEXT: [[Q5:%.*]] = load i8, ptr addrspace(1) [[MP5]], align 1
+; GFX9-NEXT: [[MP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 6
+; GFX9-NEXT: [[Q6:%.*]] = load i8, ptr addrspace(1) [[MP6]], align 2
+; GFX9-NEXT: [[MP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 7
+; GFX9-NEXT: [[Q7:%.*]] = load i8, ptr addrspace(1) [[MP7]], align 1
+; GFX9-NEXT: [[X0:%.*]] = fpext half [[V0]] to float
+; GFX9-NEXT: [[X1:%.*]] = fpext half [[V1]] to float
+; GFX9-NEXT: [[X2:%.*]] = fpext half [[V2]] to float
+; GFX9-NEXT: [[X3:%.*]] = fpext half [[V3]] to float
+; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[MAT]], align 2
+; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i8> [[TMP0]] to <4 x float>
+; GFX9-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[ZP]], i64 0
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> zeroinitializer
+; GFX9-NEXT: [[TMP4:%.*]] = fsub contract <4 x float> [[TMP1]], [[TMP3]]
+; GFX9-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
+; GFX9-NEXT: [[M0:%.*]] = fmul contract float [[TMP5]], [[X0]]
+; GFX9-NEXT: [[A0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; GFX9-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP4]], i64 1
+; GFX9-NEXT: [[M1:%.*]] = fmul contract float [[TMP6]], [[X1]]
+; GFX9-NEXT: [[A1:%.*]] = fadd contract float [[A0]], [[M1]]
+; GFX9-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP4]], i64 2
+; GFX9-NEXT: [[M2:%.*]] = fmul contract float [[TMP7]], [[X2]]
+; GFX9-NEXT: [[A2:%.*]] = fadd contract float [[A1]], [[M2]]
+; GFX9-NEXT: [[TMP8:%.*]] = extractelement <4 x float> [[TMP4]], i64 3
+; GFX9-NEXT: [[M3:%.*]] = fmul contract float [[TMP8]], [[X3]]
+; GFX9-NEXT: [[A3:%.*]] = fadd contract float [[A2]], [[M3]]
+; GFX9-NEXT: [[X4:%.*]] = fpext half [[V4]] to float
+; GFX9-NEXT: [[F4:%.*]] = sitofp i8 [[Q4]] to float
+; GFX9-NEXT: [[D4:%.*]] = fsub contract float [[F4]], [[ZP]]
+; GFX9-NEXT: [[M4:%.*]] = fmul contract float [[D4]], [[X4]]
+; GFX9-NEXT: [[A4:%.*]] = fadd contract float [[A3]], [[M4]]
+; GFX9-NEXT: [[X5:%.*]] = fpext half [[V5]] to float
+; GFX9-NEXT: [[F5:%.*]] = sitofp i8 [[Q5]] to float
+; GFX9-NEXT: [[D5:%.*]] = fsub contract float [[F5]], [[ZP]]
+; GFX9-NEXT: [[M5:%.*]] = fmul contract float [[D5]], [[X5]]
+; GFX9-NEXT: [[A5:%.*]] = fadd contract float [[A4]], [[M5]]
+; GFX9-NEXT: [[X6:%.*]] = fpext half [[V6]] to float
+; GFX9-NEXT: [[F6:%.*]] = sitofp i8 [[Q6]] to float
+; GFX9-NEXT: [[D6:%.*]] = fsub contract float [[F6]], [[ZP]]
+; GFX9-NEXT: [[M6:%.*]] = fmul contract float [[D6]], [[X6]]
+; GFX9-NEXT: [[A6:%.*]] = fadd contract float [[A5]], [[M6]]
+; GFX9-NEXT: [[X7:%.*]] = fpext half [[V7]] to float
+; GFX9-NEXT: [[F7:%.*]] = sitofp i8 [[Q7]] to float
+; GFX9-NEXT: [[D7:%.*]] = fsub contract float [[F7]], [[ZP]]
+; GFX9-NEXT: [[M7:%.*]] = fmul contract float [[D7]], [[X7]]
+; GFX9-NEXT: [[A7:%.*]] = fadd contract float [[A6]], [[M7]]
+; GFX9-NEXT: ret float [[A7]]
+;
+; GFX1030-LABEL: define float @dequant8_ordered(
+; GFX1030-SAME: ptr addrspace(1) [[MAT:%.*]], ptr addrspace(1) [[VEC:%.*]], half [[ZPH:%.*]], float [[ACC:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[ZP:%.*]] = fpext half [[ZPH]] to float
+; GFX1030-NEXT: br label %[[BODY:.*]]
+; GFX1030: [[BODY]]:
+; GFX1030-NEXT: [[V0:%.*]] = load half, ptr addrspace(1) [[VEC]], align 16
+; GFX1030-NEXT: [[VP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 2
+; GFX1030-NEXT: [[V1:%.*]] = load half, ptr addrspace(1) [[VP1]], align 2
+; GFX1030-NEXT: [[VP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 4
+; GFX1030-NEXT: [[V2:%.*]] = load half, ptr addrspace(1) [[VP2]], align 4
+; GFX1030-NEXT: [[VP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 6
+; GFX1030-NEXT: [[V3:%.*]] = load half, ptr addrspace(1) [[VP3]], align 2
+; GFX1030-NEXT: [[VP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 8
+; GFX1030-NEXT: [[V4:%.*]] = load half, ptr addrspace(1) [[VP4]], align 8
+; GFX1030-NEXT: [[VP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 10
+; GFX1030-NEXT: [[V5:%.*]] = load half, ptr addrspace(1) [[VP5]], align 2
+; GFX1030-NEXT: [[VP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 12
+; GFX1030-NEXT: [[V6:%.*]] = load half, ptr addrspace(1) [[VP6]], align 4
+; GFX1030-NEXT: [[VP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[VEC]], i64 14
+; GFX1030-NEXT: [[V7:%.*]] = load half, ptr addrspace(1) [[VP7]], align 2
+; GFX1030-NEXT: [[Q0:%.*]] = load i8, ptr addrspace(1) [[MAT]], align 2
+; GFX1030-NEXT: [[MP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 1
+; GFX1030-NEXT: [[Q1:%.*]] = load i8, ptr addrspace(1) [[MP1]], align 1
+; GFX1030-NEXT: [[MP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 2
+; GFX1030-NEXT: [[Q2:%.*]] = load i8, ptr addrspace(1) [[MP2]], align 2
+; GFX1030-NEXT: [[MP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 3
+; GFX1030-NEXT: [[Q3:%.*]] = load i8, ptr addrspace(1) [[MP3]], align 1
+; GFX1030-NEXT: [[MP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 4
+; GFX1030-NEXT: [[Q4:%.*]] = load i8, ptr addrspace(1) [[MP4]], align 2
+; GFX1030-NEXT: [[MP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 5
+; GFX1030-NEXT: [[Q5:%.*]] = load i8, ptr addrspace(1) [[MP5]], align 1
+; GFX1030-NEXT: [[MP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 6
+; GFX1030-NEXT: [[Q6:%.*]] = load i8, ptr addrspace(1) [[MP6]], align 2
+; GFX1030-NEXT: [[MP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[MAT]], i64 7
+; GFX1030-NEXT: [[Q7:%.*]] = load i8, ptr addrspace(1) [[MP7]], align 1
+; GFX1030-NEXT: [[X0:%.*]] = fpext half [[V0]] to float
+; GFX1030-NEXT: [[F0:%.*]] = sitofp i8 [[Q0]] to float
+; GFX1030-NEXT: [[D0:%.*]] = fsub contract float [[F0]], [[ZP]]
+; GFX1030-NEXT: [[M0:%.*]] = fmul contract float [[D0]], [[X0]]
+; GFX1030-NEXT: [[A0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; GFX1030-NEXT: [[X1:%.*]] = fpext half [[V1]] to float
+; GFX1030-NEXT: [[F1:%.*]] = sitofp i8 [[Q1]] to float
+; GFX1030-NEXT: [[D1:%.*]] = fsub contract float [[F1]], [[ZP]]
+; GFX1030-NEXT: [[M1:%.*]] = fmul contract float [[D1]], [[X1]]
+; GFX1030-NEXT: [[A1:%.*]] = fadd contract float [[A0]], [[M1]]
+; GFX1030-NEXT: [[X2:%.*]] = fpext half [[V2]] to float
+; GFX1030-NEXT: [[F2:%.*]] = sitofp i8 [[Q2]] to float
+; GFX1030-NEXT: [[D2:%.*]] = fsub contract float [[F2]], [[ZP]]
+; GFX1030-NEXT: [[M2:%.*]] = fmul contract float [[D2]], [[X2]]
+; GFX1030-NEXT: [[A2:%.*]] = fadd contract float [[A1]], [[M2]]
+; GFX1030-NEXT: [[X3:%.*]] = fpext half [[V3]] to float
+; GFX1030-NEXT: [[F3:%.*]] = sitofp i8 [[Q3]] to float
+; GFX1030-NEXT: [[D3:%.*]] = fsub contract float [[F3]], [[ZP]]
+; GFX1030-NEXT: [[M3:%.*]] = fmul contract float [[D3]], [[X3]]
+; GFX1030-NEXT: [[A3:%.*]] = fadd contract float [[A2]], [[M3]]
+; GFX1030-NEXT: [[X4:%.*]] = fpext half [[V4]] to float
+; GFX1030-NEXT: [[F4:%.*]] = sitofp i8 [[Q4]] to float
+; GFX1030-NEXT: [[D4:%.*]] = fsub contract float [[F4]], [[ZP]]
+; GFX1030-NEXT: [[M4:%.*]] = fmul contract float [[D4]], [[X4]]
+; GFX1030-NEXT: [[A4:%.*]] = fadd contract float [[A3]], [[M4]]
+; GFX1030-NEXT: [[X5:%.*]] = fpext half [[V5]] to float
+; GFX1030-NEXT: [[F5:%.*]] = sitofp i8 [[Q5]] to float
+; GFX1030-NEXT: [[D5:%.*]] = fsub contract float [[F5]], [[ZP]]
+; GFX1030-NEXT: [[M5:%.*]] = fmul contract float [[D5]], [[X5]]
+; GFX1030-NEXT: [[A5:%.*]] = fadd contract float [[A4]], [[M5]]
+; GFX1030-NEXT: [[X6:%.*]] = fpext half [[V6]] to float
+; GFX1030-NEXT: [[F6:%.*]] = sitofp i8 [[Q6]] to float
+; GFX1030-NEXT: [[D6:%.*]] = fsub contract float [[F6]], [[ZP]]
+; GFX1030-NEXT: [[M6:%.*]] = fmul contract float [[D6]], [[X6]]
+; GFX1030-NEXT: [[A6:%.*]] = fadd contract float [[A5]], [[M6]]
+; GFX1030-NEXT: [[X7:%.*]] = fpext half [[V7]] to float
+; GFX1030-NEXT: [[F7:%.*]] = sitofp i8 [[Q7]] to float
+; GFX1030-NEXT: [[D7:%.*]] = fsub contract float [[F7]], [[ZP]]
+; GFX1030-NEXT: [[M7:%.*]] = fmul contract float [[D7]], [[X7]]
+; GFX1030-NEXT: [[A7:%.*]] = fadd contract float [[A6]], [[M7]]
+; GFX1030-NEXT: ret float [[A7]]
;
entry:
%zp = fpext half %zph to float
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
index 3a41dc15103c15..0e64c7ca921288 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
@@ -17,40 +17,99 @@ define amdgpu_kernel void @dotconv(ptr addrspace(1) noalias readonly align 4 %in
; CHECK-NEXT: [[SEXT_I:%.*]] = shl i64 [[CALL_I]], 32
; CHECK-NEXT: [[IDX_EXT_I:%.*]] = ashr exact i64 [[SEXT_I]], 32
; CHECK-NEXT: [[ADD_PTR_I:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(1) [[IN]], i64 [[IDX_EXT_I]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr addrspace(1) [[ADD_PTR_I]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr addrspace(1) [[MASK]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <16 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[MUL_I:%.*]] = extractelement <16 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr addrspace(1) [[ADD_PTR_I]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[MASK]], align 4
+; CHECK-NEXT: [[MUL_I:%.*]] = fmul contract float [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[ADD_I:%.*]] = fadd contract float [[MUL_I]], 0.000000e+00
-; CHECK-NEXT: [[MUL_I_1:%.*]] = extractelement <16 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[ARRAYIDX_I_1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 4
+; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_1]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 4
+; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_1]], align 4
+; CHECK-NEXT: [[MUL_I_1:%.*]] = fmul contract float [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[ADD_I_1:%.*]] = fadd contract float [[ADD_I]], [[MUL_I_1]]
-; CHECK-NEXT: [[MUL_I_2:%.*]] = extractelement <16 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[ARRAYIDX_I_2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 8
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_2]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 8
+; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_2]], align 4
+; CHECK-NEXT: [[MUL_I_2:%.*]] = fmul contract float [[TMP4]], [[TMP5]]
; CHECK-NEXT: [[ADD_I_2:%.*]] = fadd contract float [[ADD_I_1]], [[MUL_I_2]]
-; CHECK-NEXT: [[MUL_I_3:%.*]] = extractelement <16 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[ARRAYIDX_I_3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 12
+; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_3]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 12
+; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_3]], align 4
+; CHECK-NEXT: [[MUL_I_3:%.*]] = fmul contract float [[TMP6]], [[TMP7]]
; CHECK-NEXT: [[ADD_I_3:%.*]] = fadd contract float [[ADD_I_2]], [[MUL_I_3]]
-; CHECK-NEXT: [[MUL_I_4:%.*]] = extractelement <16 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[ARRAYIDX_I_4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 16
+; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_4]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 16
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_4]], align 4
+; CHECK-NEXT: [[MUL_I_4:%.*]] = fmul contract float [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[ADD_I_4:%.*]] = fadd contract float [[ADD_I_3]], [[MUL_I_4]]
-; CHECK-NEXT: [[MUL_I_5:%.*]] = extractelement <16 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[ARRAYIDX_I_5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 20
+; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_5]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 20
+; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_5]], align 4
+; CHECK-NEXT: [[MUL_I_5:%.*]] = fmul contract float [[TMP10]], [[TMP11]]
; CHECK-NEXT: [[ADD_I_5:%.*]] = fadd contract float [[ADD_I_4]], [[MUL_I_5]]
-; CHECK-NEXT: [[MUL_I_6:%.*]] = extractelement <16 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[ARRAYIDX_I_6:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 24
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_6]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_6:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 24
+; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_6]], align 4
+; CHECK-NEXT: [[MUL_I_6:%.*]] = fmul contract float [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[ADD_I_6:%.*]] = fadd contract float [[ADD_I_5]], [[MUL_I_6]]
-; CHECK-NEXT: [[MUL_I_7:%.*]] = extractelement <16 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[ARRAYIDX_I_7:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 28
+; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_7]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_7:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 28
+; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_7]], align 4
+; CHECK-NEXT: [[MUL_I_7:%.*]] = fmul contract float [[TMP14]], [[TMP15]]
; CHECK-NEXT: [[ADD_I_7:%.*]] = fadd contract float [[ADD_I_6]], [[MUL_I_7]]
-; CHECK-NEXT: [[MUL_I_8:%.*]] = extractelement <16 x float> [[TMP2]], i64 8
+; CHECK-NEXT: [[ARRAYIDX_I_8:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 32
+; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_8]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_8:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 32
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_8]], align 4
+; CHECK-NEXT: [[MUL_I_8:%.*]] = fmul contract float [[TMP16]], [[TMP17]]
; CHECK-NEXT: [[ADD_I_8:%.*]] = fadd contract float [[ADD_I_7]], [[MUL_I_8]]
-; CHECK-NEXT: [[MUL_I_9:%.*]] = extractelement <16 x float> [[TMP2]], i64 9
+; CHECK-NEXT: [[ARRAYIDX_I_9:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 36
+; CHECK-NEXT: [[TMP18:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_9]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_9:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 36
+; CHECK-NEXT: [[TMP19:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_9]], align 4
+; CHECK-NEXT: [[MUL_I_9:%.*]] = fmul contract float [[TMP18]], [[TMP19]]
; CHECK-NEXT: [[ADD_I_9:%.*]] = fadd contract float [[ADD_I_8]], [[MUL_I_9]]
-; CHECK-NEXT: [[MUL_I_10:%.*]] = extractelement <16 x float> [[TMP2]], i64 10
+; CHECK-NEXT: [[ARRAYIDX_I_10:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 40
+; CHECK-NEXT: [[TMP20:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_10]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_10:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 40
+; CHECK-NEXT: [[TMP21:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_10]], align 4
+; CHECK-NEXT: [[MUL_I_10:%.*]] = fmul contract float [[TMP20]], [[TMP21]]
; CHECK-NEXT: [[ADD_I_10:%.*]] = fadd contract float [[ADD_I_9]], [[MUL_I_10]]
-; CHECK-NEXT: [[MUL_I_11:%.*]] = extractelement <16 x float> [[TMP2]], i64 11
+; CHECK-NEXT: [[ARRAYIDX_I_11:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 44
+; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_11]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_11:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 44
+; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_11]], align 4
+; CHECK-NEXT: [[MUL_I_11:%.*]] = fmul contract float [[TMP22]], [[TMP23]]
; CHECK-NEXT: [[ADD_I_11:%.*]] = fadd contract float [[ADD_I_10]], [[MUL_I_11]]
-; CHECK-NEXT: [[MUL_I_12:%.*]] = extractelement <16 x float> [[TMP2]], i64 12
+; CHECK-NEXT: [[ARRAYIDX_I_12:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 48
+; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_12]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_12:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 48
+; CHECK-NEXT: [[TMP25:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_12]], align 4
+; CHECK-NEXT: [[MUL_I_12:%.*]] = fmul contract float [[TMP24]], [[TMP25]]
; CHECK-NEXT: [[ADD_I_12:%.*]] = fadd contract float [[ADD_I_11]], [[MUL_I_12]]
-; CHECK-NEXT: [[MUL_I_13:%.*]] = extractelement <16 x float> [[TMP2]], i64 13
+; CHECK-NEXT: [[ARRAYIDX_I_13:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 52
+; CHECK-NEXT: [[TMP26:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_13]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_13:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 52
+; CHECK-NEXT: [[TMP27:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_13]], align 4
+; CHECK-NEXT: [[MUL_I_13:%.*]] = fmul contract float [[TMP26]], [[TMP27]]
; CHECK-NEXT: [[ADD_I_13:%.*]] = fadd contract float [[ADD_I_12]], [[MUL_I_13]]
-; CHECK-NEXT: [[MUL_I_14:%.*]] = extractelement <16 x float> [[TMP2]], i64 14
+; CHECK-NEXT: [[ARRAYIDX_I_14:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 56
+; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_14]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_14:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 56
+; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_14]], align 4
+; CHECK-NEXT: [[MUL_I_14:%.*]] = fmul contract float [[TMP28]], [[TMP29]]
; CHECK-NEXT: [[ADD_I_14:%.*]] = fadd contract float [[ADD_I_13]], [[MUL_I_14]]
-; CHECK-NEXT: [[MUL_I_15:%.*]] = extractelement <16 x float> [[TMP2]], i64 15
+; CHECK-NEXT: [[ARRAYIDX_I_15:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[ADD_PTR_I]], i64 60
+; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX_I_15]], align 4
+; CHECK-NEXT: [[ARRAYIDX3_I_15:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[MASK]], i64 60
+; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX3_I_15]], align 4
+; CHECK-NEXT: [[MUL_I_15:%.*]] = fmul contract float [[TMP30]], [[TMP31]]
; CHECK-NEXT: [[ADD_I_15:%.*]] = fadd contract float [[ADD_I_14]], [[MUL_I_15]]
; CHECK-NEXT: [[ARRAYIDX5_I:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(1) [[OUT]], i64 [[IDX_EXT_I]]
; CHECK-NEXT: store float [[ADD_I_15]], ptr addrspace(1) [[ARRAYIDX5_I]], align 4
@@ -167,70 +226,36 @@ declare i64 @_Z13get_global_idj(i32 noundef)
; coalesced loads and stay scalar too.
define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
-; GFX9-LABEL: define float @dot_i16_window(
-; GFX9-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX9-NEXT: [[ENTRY:.*:]]
-; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; GFX9-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; GFX9-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; GFX9-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; GFX9-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
-; GFX9-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
-; GFX9-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
-; GFX9-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
-; GFX9-NEXT: ret float [[S3]]
-;
-; GFX1030-LABEL: define float @dot_i16_window(
-; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX1030-NEXT: [[ENTRY:.*:]]
-; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
-; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
-; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
-; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
-; GFX1030-NEXT: ret float [[S3]]
-;
-; GFX1250-LABEL: define float @dot_i16_window(
-; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX1250-NEXT: [[ENTRY:.*:]]
-; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
-; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
-; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
-; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
-; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
-; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
-; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
-; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
-; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
-; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
-; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
-; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
-; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
-; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
-; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
-; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
-; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
-; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
-; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
-; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
-; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
-; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
-; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
-; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
-; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
-; GFX1250-NEXT: ret float [[S3]]
+; CHECK-LABEL: define float @dot_i16_window(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
+; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
+; CHECK-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
+; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
+; CHECK-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
+; CHECK-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
+; CHECK-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
+; CHECK-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
+; CHECK-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
+; CHECK-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
+; CHECK-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
+; CHECK-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
+; CHECK-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
+; CHECK-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
+; CHECK-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
+; CHECK-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
+; CHECK-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
+; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
+; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; CHECK-NEXT: ret float [[S3]]
;
entry:
%a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
@@ -276,24 +301,51 @@ define float @dot8_ordered_loop(ptr addrspace(1) noalias readonly align 4 %a0, p
; CHECK-NEXT: [[OFF:%.*]] = shl i64 [[I]], 3
; CHECK-NEXT: [[A:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A0]], i64 [[OFF]]
; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B0]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[M0:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[LA0:%.*]] = load float, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[LB0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float [[LA0]], [[LB0]]
; CHECK-NEXT: [[R0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; CHECK-NEXT: [[M1:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 1
+; CHECK-NEXT: [[LA1:%.*]] = load float, ptr addrspace(1) [[PA1]], align 4
+; CHECK-NEXT: [[PB1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 1
+; CHECK-NEXT: [[LB1:%.*]] = load float, ptr addrspace(1) [[PB1]], align 4
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float [[LA1]], [[LB1]]
; CHECK-NEXT: [[R1:%.*]] = fadd contract float [[R0]], [[M1]]
-; CHECK-NEXT: [[M2:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 2
+; CHECK-NEXT: [[LA2:%.*]] = load float, ptr addrspace(1) [[PA2]], align 4
+; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 2
+; CHECK-NEXT: [[LB2:%.*]] = load float, ptr addrspace(1) [[PB2]], align 4
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float [[LA2]], [[LB2]]
; CHECK-NEXT: [[R2:%.*]] = fadd contract float [[R1]], [[M2]]
-; CHECK-NEXT: [[M3:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 3
+; CHECK-NEXT: [[LA3:%.*]] = load float, ptr addrspace(1) [[PA3]], align 4
+; CHECK-NEXT: [[PB3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 3
+; CHECK-NEXT: [[LB3:%.*]] = load float, ptr addrspace(1) [[PB3]], align 4
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float [[LA3]], [[LB3]]
; CHECK-NEXT: [[R3:%.*]] = fadd contract float [[R2]], [[M3]]
-; CHECK-NEXT: [[M4:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 4
+; CHECK-NEXT: [[LA4:%.*]] = load float, ptr addrspace(1) [[PA4]], align 4
+; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 4
+; CHECK-NEXT: [[LB4:%.*]] = load float, ptr addrspace(1) [[PB4]], align 4
+; CHECK-NEXT: [[M4:%.*]] = fmul contract float [[LA4]], [[LB4]]
; CHECK-NEXT: [[R4:%.*]] = fadd contract float [[R3]], [[M4]]
-; CHECK-NEXT: [[M5:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[PA5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 5
+; CHECK-NEXT: [[LA5:%.*]] = load float, ptr addrspace(1) [[PA5]], align 4
+; CHECK-NEXT: [[PB5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 5
+; CHECK-NEXT: [[LB5:%.*]] = load float, ptr addrspace(1) [[PB5]], align 4
+; CHECK-NEXT: [[M5:%.*]] = fmul contract float [[LA5]], [[LB5]]
; CHECK-NEXT: [[R5:%.*]] = fadd contract float [[R4]], [[M5]]
-; CHECK-NEXT: [[M6:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[PA6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 6
+; CHECK-NEXT: [[LA6:%.*]] = load float, ptr addrspace(1) [[PA6]], align 4
+; CHECK-NEXT: [[PB6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 6
+; CHECK-NEXT: [[LB6:%.*]] = load float, ptr addrspace(1) [[PB6]], align 4
+; CHECK-NEXT: [[M6:%.*]] = fmul contract float [[LA6]], [[LB6]]
; CHECK-NEXT: [[R6:%.*]] = fadd contract float [[R5]], [[M6]]
-; CHECK-NEXT: [[M7:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[PA7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 7
+; CHECK-NEXT: [[LA7:%.*]] = load float, ptr addrspace(1) [[PA7]], align 4
+; CHECK-NEXT: [[PB7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 7
+; CHECK-NEXT: [[LB7:%.*]] = load float, ptr addrspace(1) [[PB7]], align 4
+; CHECK-NEXT: [[M7:%.*]] = fmul contract float [[LA7]], [[LB7]]
; CHECK-NEXT: [[R7]] = fadd contract float [[R6]], [[M7]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], 16
@@ -372,25 +424,52 @@ define float @dot8_ordered_lds_a16(ptr addrspace(3) %a, ptr addrspace(3) %b, flo
; CHECK-SAME: ptr addrspace(3) [[A:%.*]], ptr addrspace(3) [[B:%.*]], float [[ACC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 0
+; CHECK-NEXT: [[LA0:%.*]] = load float, ptr addrspace(3) [[PA0]], align 16
; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 0
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(3) [[PA0]], align 16
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(3) [[PB0]], align 16
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[LB0:%.*]] = load float, ptr addrspace(3) [[PB0]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract float [[LA0]], [[LB0]]
; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[PA1:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 1
+; CHECK-NEXT: [[LA1:%.*]] = load float, ptr addrspace(3) [[PA1]], align 4
+; CHECK-NEXT: [[PB1:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 1
+; CHECK-NEXT: [[LB1:%.*]] = load float, ptr addrspace(3) [[PB1]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract float [[LA1]], [[LB1]]
; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[PA2:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 2
+; CHECK-NEXT: [[LA2:%.*]] = load float, ptr addrspace(3) [[PA2]], align 8
+; CHECK-NEXT: [[PB2:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 2
+; CHECK-NEXT: [[LB2:%.*]] = load float, ptr addrspace(3) [[PB2]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fmul contract float [[LA2]], [[LB2]]
; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[PA3:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 3
+; CHECK-NEXT: [[LA3:%.*]] = load float, ptr addrspace(3) [[PA3]], align 4
+; CHECK-NEXT: [[PB3:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 3
+; CHECK-NEXT: [[LB3:%.*]] = load float, ptr addrspace(3) [[PB3]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = fmul contract float [[LA3]], [[LB3]]
; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; CHECK-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 4
+; CHECK-NEXT: [[LA4:%.*]] = load float, ptr addrspace(3) [[PA4]], align 16
+; CHECK-NEXT: [[PB4:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 4
+; CHECK-NEXT: [[LB4:%.*]] = load float, ptr addrspace(3) [[PB4]], align 16
+; CHECK-NEXT: [[TMP7:%.*]] = fmul contract float [[LA4]], [[LB4]]
; CHECK-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; CHECK-NEXT: [[PA5:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 5
+; CHECK-NEXT: [[LA5:%.*]] = load float, ptr addrspace(3) [[PA5]], align 4
+; CHECK-NEXT: [[PB5:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 5
+; CHECK-NEXT: [[LB5:%.*]] = load float, ptr addrspace(3) [[PB5]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract float [[LA5]], [[LB5]]
; CHECK-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; CHECK-NEXT: [[PA6:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 6
+; CHECK-NEXT: [[LA6:%.*]] = load float, ptr addrspace(3) [[PA6]], align 8
+; CHECK-NEXT: [[PB6:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 6
+; CHECK-NEXT: [[LB6:%.*]] = load float, ptr addrspace(3) [[PB6]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = fmul contract float [[LA6]], [[LB6]]
; CHECK-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; CHECK-NEXT: [[PA7:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[A]], i32 7
+; CHECK-NEXT: [[LA7:%.*]] = load float, ptr addrspace(3) [[PA7]], align 4
+; CHECK-NEXT: [[PB7:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[B]], i32 7
+; CHECK-NEXT: [[LB7:%.*]] = load float, ptr addrspace(3) [[PB7]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = fmul contract float [[LA7]], [[LB7]]
; CHECK-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP10]]
; CHECK-NEXT: ret float [[S7]]
;
@@ -454,30 +533,134 @@ entry:
; fmul lane by lane and every lane still fuses with its fadd, so it keeps the
; vector form.
define float @dot8_ordered_noacc(ptr addrspace(1) noalias readonly align 4 %a, ptr addrspace(1) noalias readonly align 4 %b) {
-; CHECK-LABEL: define float @dot8_ordered_noacc(
-; CHECK-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
-; CHECK-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[PA0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PB0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
-; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
-; CHECK-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
-; CHECK-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
-; CHECK-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
-; CHECK-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP10]]
-; CHECK-NEXT: ret float [[S7]]
+; GFX9-LABEL: define float @dot8_ordered_noacc(
+; GFX9-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; GFX9-NEXT: [[LA0:%.*]] = load float, ptr addrspace(1) [[PA0]], align 4
+; GFX9-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; GFX9-NEXT: [[LB0:%.*]] = load float, ptr addrspace(1) [[PB0]], align 4
+; GFX9-NEXT: [[M0:%.*]] = fmul contract float [[LA0]], [[LB0]]
+; GFX9-NEXT: [[PA1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 1
+; GFX9-NEXT: [[LA1:%.*]] = load float, ptr addrspace(1) [[PA1]], align 4
+; GFX9-NEXT: [[PB1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 1
+; GFX9-NEXT: [[LB1:%.*]] = load float, ptr addrspace(1) [[PB1]], align 4
+; GFX9-NEXT: [[M1:%.*]] = fmul contract float [[LA1]], [[LB1]]
+; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[M0]], [[M1]]
+; GFX9-NEXT: [[PA2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 2
+; GFX9-NEXT: [[LA2:%.*]] = load float, ptr addrspace(1) [[PA2]], align 4
+; GFX9-NEXT: [[PB2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 2
+; GFX9-NEXT: [[LB2:%.*]] = load float, ptr addrspace(1) [[PB2]], align 4
+; GFX9-NEXT: [[M2:%.*]] = fmul contract float [[LA2]], [[LB2]]
+; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX9-NEXT: [[PA3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 3
+; GFX9-NEXT: [[LA3:%.*]] = load float, ptr addrspace(1) [[PA3]], align 4
+; GFX9-NEXT: [[PB3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 3
+; GFX9-NEXT: [[LB3:%.*]] = load float, ptr addrspace(1) [[PB3]], align 4
+; GFX9-NEXT: [[M3:%.*]] = fmul contract float [[LA3]], [[LB3]]
+; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX9-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 4
+; GFX9-NEXT: [[LA4:%.*]] = load float, ptr addrspace(1) [[PA4]], align 4
+; GFX9-NEXT: [[PB4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 4
+; GFX9-NEXT: [[LB4:%.*]] = load float, ptr addrspace(1) [[PB4]], align 4
+; GFX9-NEXT: [[M4:%.*]] = fmul contract float [[LA4]], [[LB4]]
+; GFX9-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[M4]]
+; GFX9-NEXT: [[PA5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 5
+; GFX9-NEXT: [[LA5:%.*]] = load float, ptr addrspace(1) [[PA5]], align 4
+; GFX9-NEXT: [[PB5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 5
+; GFX9-NEXT: [[LB5:%.*]] = load float, ptr addrspace(1) [[PB5]], align 4
+; GFX9-NEXT: [[M5:%.*]] = fmul contract float [[LA5]], [[LB5]]
+; GFX9-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[M5]]
+; GFX9-NEXT: [[PA6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 6
+; GFX9-NEXT: [[LA6:%.*]] = load float, ptr addrspace(1) [[PA6]], align 4
+; GFX9-NEXT: [[PB6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 6
+; GFX9-NEXT: [[LB6:%.*]] = load float, ptr addrspace(1) [[PB6]], align 4
+; GFX9-NEXT: [[M6:%.*]] = fmul contract float [[LA6]], [[LB6]]
+; GFX9-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[M6]]
+; GFX9-NEXT: [[PA7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 7
+; GFX9-NEXT: [[LA7:%.*]] = load float, ptr addrspace(1) [[PA7]], align 4
+; GFX9-NEXT: [[PB7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 7
+; GFX9-NEXT: [[LB7:%.*]] = load float, ptr addrspace(1) [[PB7]], align 4
+; GFX9-NEXT: [[M7:%.*]] = fmul contract float [[LA7]], [[LB7]]
+; GFX9-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[M7]]
+; GFX9-NEXT: ret float [[S7]]
+;
+; GFX1030-LABEL: define float @dot8_ordered_noacc(
+; GFX1030-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; GFX1030-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; GFX1030-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr addrspace(1) [[PA0]], align 4
+; GFX1030-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PB0]], align 4
+; GFX1030-NEXT: [[TMP2:%.*]] = fmul contract <8 x float> [[TMP0]], [[TMP1]]
+; GFX1030-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP2]], i64 0
+; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP2]], i64 1
+; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[TMP3]], [[TMP4]]
+; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP2]], i64 2
+; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP5]]
+; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP2]], i64 3
+; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP6]]
+; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP2]], i64 4
+; GFX1030-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP7]]
+; GFX1030-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP2]], i64 5
+; GFX1030-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP8]]
+; GFX1030-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP2]], i64 6
+; GFX1030-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP9]]
+; GFX1030-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP2]], i64 7
+; GFX1030-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP10]]
+; GFX1030-NEXT: ret float [[S7]]
+;
+; GFX1250-LABEL: define float @dot8_ordered_noacc(
+; GFX1250-SAME: ptr addrspace(1) noalias readonly align 4 [[A:%.*]], ptr addrspace(1) noalias readonly align 4 [[B:%.*]]) {
+; GFX1250-NEXT: [[ENTRY:.*:]]
+; GFX1250-NEXT: [[PA0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 0
+; GFX1250-NEXT: [[LA0:%.*]] = load float, ptr addrspace(1) [[PA0]], align 4
+; GFX1250-NEXT: [[PB0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 0
+; GFX1250-NEXT: [[LB0:%.*]] = load float, ptr addrspace(1) [[PB0]], align 4
+; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[LA0]], [[LB0]]
+; GFX1250-NEXT: [[PA1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 1
+; GFX1250-NEXT: [[LA1:%.*]] = load float, ptr addrspace(1) [[PA1]], align 4
+; GFX1250-NEXT: [[PB1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 1
+; GFX1250-NEXT: [[LB1:%.*]] = load float, ptr addrspace(1) [[PB1]], align 4
+; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[LA1]], [[LB1]]
+; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[M0]], [[M1]]
+; GFX1250-NEXT: [[PA2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 2
+; GFX1250-NEXT: [[LA2:%.*]] = load float, ptr addrspace(1) [[PA2]], align 4
+; GFX1250-NEXT: [[PB2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 2
+; GFX1250-NEXT: [[LB2:%.*]] = load float, ptr addrspace(1) [[PB2]], align 4
+; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[LA2]], [[LB2]]
+; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX1250-NEXT: [[PA3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 3
+; GFX1250-NEXT: [[LA3:%.*]] = load float, ptr addrspace(1) [[PA3]], align 4
+; GFX1250-NEXT: [[PB3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 3
+; GFX1250-NEXT: [[LB3:%.*]] = load float, ptr addrspace(1) [[PB3]], align 4
+; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[LA3]], [[LB3]]
+; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX1250-NEXT: [[PA4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 4
+; GFX1250-NEXT: [[LA4:%.*]] = load float, ptr addrspace(1) [[PA4]], align 4
+; GFX1250-NEXT: [[PB4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 4
+; GFX1250-NEXT: [[LB4:%.*]] = load float, ptr addrspace(1) [[PB4]], align 4
+; GFX1250-NEXT: [[M4:%.*]] = fmul contract float [[LA4]], [[LB4]]
+; GFX1250-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[M4]]
+; GFX1250-NEXT: [[PA5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 5
+; GFX1250-NEXT: [[LA5:%.*]] = load float, ptr addrspace(1) [[PA5]], align 4
+; GFX1250-NEXT: [[PB5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 5
+; GFX1250-NEXT: [[LB5:%.*]] = load float, ptr addrspace(1) [[PB5]], align 4
+; GFX1250-NEXT: [[M5:%.*]] = fmul contract float [[LA5]], [[LB5]]
+; GFX1250-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[M5]]
+; GFX1250-NEXT: [[PA6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 6
+; GFX1250-NEXT: [[LA6:%.*]] = load float, ptr addrspace(1) [[PA6]], align 4
+; GFX1250-NEXT: [[PB6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 6
+; GFX1250-NEXT: [[LB6:%.*]] = load float, ptr addrspace(1) [[PB6]], align 4
+; GFX1250-NEXT: [[M6:%.*]] = fmul contract float [[LA6]], [[LB6]]
+; GFX1250-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[M6]]
+; GFX1250-NEXT: [[PA7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 7
+; GFX1250-NEXT: [[LA7:%.*]] = load float, ptr addrspace(1) [[PA7]], align 4
+; GFX1250-NEXT: [[PB7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[B]], i64 7
+; GFX1250-NEXT: [[LB7:%.*]] = load float, ptr addrspace(1) [[PB7]], align 4
+; GFX1250-NEXT: [[M7:%.*]] = fmul contract float [[LA7]], [[LB7]]
+; GFX1250-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[M7]]
+; GFX1250-NEXT: ret float [[S7]]
;
entry:
%pa0 = getelementptr inbounds float, ptr addrspace(1) %a, i64 0
@@ -541,26 +724,67 @@ define float @dot8_sitofp_i16(ptr addrspace(1) noalias readonly align 16 %a, ptr
; GFX9-SAME: ptr addrspace(1) noalias readonly align 16 [[A:%.*]], ptr addrspace(1) noalias readonly align 16 [[B:%.*]]) {
; GFX9-NEXT: [[ENTRY:.*:]]
; GFX9-NEXT: [[PA0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 0
+; GFX9-NEXT: [[LA0:%.*]] = load i16, ptr addrspace(1) [[PA0]], align 16
+; GFX9-NEXT: [[FA0:%.*]] = sitofp i16 [[LA0]] to float
; GFX9-NEXT: [[PB0:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 0
-; GFX9-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr addrspace(1) [[PA0]], align 16
-; GFX9-NEXT: [[TMP1:%.*]] = sitofp <8 x i16> [[TMP0]] to <8 x float>
-; GFX9-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr addrspace(1) [[PB0]], align 16
-; GFX9-NEXT: [[TMP3:%.*]] = sitofp <8 x i16> [[TMP2]] to <8 x float>
-; GFX9-NEXT: [[TMP4:%.*]] = fmul contract <8 x float> [[TMP1]], [[TMP3]]
-; GFX9-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP4]], i64 0
-; GFX9-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP4]], i64 1
+; GFX9-NEXT: [[LB0:%.*]] = load i16, ptr addrspace(1) [[PB0]], align 16
+; GFX9-NEXT: [[FB0:%.*]] = sitofp i16 [[LB0]] to float
+; GFX9-NEXT: [[TMP5:%.*]] = fmul contract float [[FA0]], [[FB0]]
+; GFX9-NEXT: [[PA1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 1
+; GFX9-NEXT: [[LA1:%.*]] = load i16, ptr addrspace(1) [[PA1]], align 2
+; GFX9-NEXT: [[FA1:%.*]] = sitofp i16 [[LA1]] to float
+; GFX9-NEXT: [[PB1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 1
+; GFX9-NEXT: [[LB1:%.*]] = load i16, ptr addrspace(1) [[PB1]], align 2
+; GFX9-NEXT: [[FB1:%.*]] = sitofp i16 [[LB1]] to float
+; GFX9-NEXT: [[TMP6:%.*]] = fmul contract float [[FA1]], [[FB1]]
; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[TMP5]], [[TMP6]]
-; GFX9-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP4]], i64 2
+; GFX9-NEXT: [[PA2:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 2
+; GFX9-NEXT: [[LA2:%.*]] = load i16, ptr addrspace(1) [[PA2]], align 4
+; GFX9-NEXT: [[FA2:%.*]] = sitofp i16 [[LA2]] to float
+; GFX9-NEXT: [[PB2:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 2
+; GFX9-NEXT: [[LB2:%.*]] = load i16, ptr addrspace(1) [[PB2]], align 4
+; GFX9-NEXT: [[FB2:%.*]] = sitofp i16 [[LB2]] to float
+; GFX9-NEXT: [[TMP7:%.*]] = fmul contract float [[FA2]], [[FB2]]
; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP7]]
-; GFX9-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP4]], i64 3
+; GFX9-NEXT: [[PA3:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 3
+; GFX9-NEXT: [[LA3:%.*]] = load i16, ptr addrspace(1) [[PA3]], align 2
+; GFX9-NEXT: [[FA3:%.*]] = sitofp i16 [[LA3]] to float
+; GFX9-NEXT: [[PB3:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 3
+; GFX9-NEXT: [[LB3:%.*]] = load i16, ptr addrspace(1) [[PB3]], align 2
+; GFX9-NEXT: [[FB3:%.*]] = sitofp i16 [[LB3]] to float
+; GFX9-NEXT: [[TMP8:%.*]] = fmul contract float [[FA3]], [[FB3]]
; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP8]]
-; GFX9-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP4]], i64 4
+; GFX9-NEXT: [[PA4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 4
+; GFX9-NEXT: [[LA4:%.*]] = load i16, ptr addrspace(1) [[PA4]], align 8
+; GFX9-NEXT: [[FA4:%.*]] = sitofp i16 [[LA4]] to float
+; GFX9-NEXT: [[PB4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 4
+; GFX9-NEXT: [[LB4:%.*]] = load i16, ptr addrspace(1) [[PB4]], align 8
+; GFX9-NEXT: [[FB4:%.*]] = sitofp i16 [[LB4]] to float
+; GFX9-NEXT: [[TMP9:%.*]] = fmul contract float [[FA4]], [[FB4]]
; GFX9-NEXT: [[S4:%.*]] = fadd contract float [[S3]], [[TMP9]]
-; GFX9-NEXT: [[TMP10:%.*]] = extractelement <8 x float> [[TMP4]], i64 5
+; GFX9-NEXT: [[PA5:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 5
+; GFX9-NEXT: [[LA5:%.*]] = load i16, ptr addrspace(1) [[PA5]], align 2
+; GFX9-NEXT: [[FA5:%.*]] = sitofp i16 [[LA5]] to float
+; GFX9-NEXT: [[PB5:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 5
+; GFX9-NEXT: [[LB5:%.*]] = load i16, ptr addrspace(1) [[PB5]], align 2
+; GFX9-NEXT: [[FB5:%.*]] = sitofp i16 [[LB5]] to float
+; GFX9-NEXT: [[TMP10:%.*]] = fmul contract float [[FA5]], [[FB5]]
; GFX9-NEXT: [[S5:%.*]] = fadd contract float [[S4]], [[TMP10]]
-; GFX9-NEXT: [[TMP11:%.*]] = extractelement <8 x float> [[TMP4]], i64 6
+; GFX9-NEXT: [[PA6:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 6
+; GFX9-NEXT: [[LA6:%.*]] = load i16, ptr addrspace(1) [[PA6]], align 4
+; GFX9-NEXT: [[FA6:%.*]] = sitofp i16 [[LA6]] to float
+; GFX9-NEXT: [[PB6:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 6
+; GFX9-NEXT: [[LB6:%.*]] = load i16, ptr addrspace(1) [[PB6]], align 4
+; GFX9-NEXT: [[FB6:%.*]] = sitofp i16 [[LB6]] to float
+; GFX9-NEXT: [[TMP11:%.*]] = fmul contract float [[FA6]], [[FB6]]
; GFX9-NEXT: [[S6:%.*]] = fadd contract float [[S5]], [[TMP11]]
-; GFX9-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[TMP4]], i64 7
+; GFX9-NEXT: [[PA7:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[A]], i64 7
+; GFX9-NEXT: [[LA7:%.*]] = load i16, ptr addrspace(1) [[PA7]], align 2
+; GFX9-NEXT: [[FA7:%.*]] = sitofp i16 [[LA7]] to float
+; GFX9-NEXT: [[PB7:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[B]], i64 7
+; GFX9-NEXT: [[LB7:%.*]] = load i16, ptr addrspace(1) [[PB7]], align 2
+; GFX9-NEXT: [[FB7:%.*]] = sitofp i16 [[LB7]] to float
+; GFX9-NEXT: [[TMP12:%.*]] = fmul contract float [[FA7]], [[FB7]]
; GFX9-NEXT: [[S7:%.*]] = fadd contract float [[S6]], [[TMP12]]
; GFX9-NEXT: ret float [[S7]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index a25409530b4bb7..acd0101e9724d5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -3,7 +3,7 @@
; gfx90a fuses a scalar fmul feeding a contractable fadd into a single fma, so
; vectorizing an ordered fadd-of-fmul reduction, a convolution here, would break
-; that fusion. The contract reduction must stay mostly scalar, while the same
+; that fusion. The contract reduction must stay scalar, while the same
; reduction without contract, where fusion was never possible, is free to vectorize.
define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
@@ -14,98 +14,148 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4
; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV4]], [[MV0]]
; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1
+; CHECK-NEXT: [[IV1:%.*]] = load float, ptr addrspace(1) [[IP0]], align 4
; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1
+; CHECK-NEXT: [[MV1:%.*]] = load float, ptr addrspace(4) [[MP0]], align 4
+; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[IV1]], [[MV1]]
+; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[PROD1]]
+; CHECK-NEXT: [[IP2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 2
+; CHECK-NEXT: [[IV2:%.*]] = load float, ptr addrspace(1) [[IP2]], align 4
+; CHECK-NEXT: [[MP2:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 2
+; CHECK-NEXT: [[MV2:%.*]] = load float, ptr addrspace(4) [[MP2]], align 4
+; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[IV2]], [[MV2]]
+; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
+; CHECK-NEXT: [[IP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 3
+; CHECK-NEXT: [[IV3:%.*]] = load float, ptr addrspace(1) [[IP3]], align 4
+; CHECK-NEXT: [[MP3:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 3
+; CHECK-NEXT: [[MV3:%.*]] = load float, ptr addrspace(4) [[MP3]], align 4
+; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[IV3]], [[MV3]]
+; CHECK-NEXT: [[ACC3:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
+; CHECK-NEXT: [[IP9:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
+; CHECK-NEXT: [[IV9:%.*]] = load float, ptr addrspace(1) [[IP9]], align 4
+; CHECK-NEXT: [[MP4:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 4
+; CHECK-NEXT: [[MV4:%.*]] = load float, ptr addrspace(4) [[MP4]], align 4
+; CHECK-NEXT: [[PROD4:%.*]] = fmul contract float [[IV9]], [[MV4]]
+; CHECK-NEXT: [[ACC4:%.*]] = fadd contract float [[ACC3]], [[PROD4]]
; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
+; CHECK-NEXT: [[IV10:%.*]] = load float, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[MP5:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 5
+; CHECK-NEXT: [[MV5:%.*]] = load float, ptr addrspace(4) [[MP5]], align 4
+; CHECK-NEXT: [[PROD5:%.*]] = fmul contract float [[IV10]], [[MV5]]
+; CHECK-NEXT: [[ACC5:%.*]] = fadd contract float [[ACC4]], [[PROD5]]
+; CHECK-NEXT: [[IP11:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
+; CHECK-NEXT: [[IV6:%.*]] = load float, ptr addrspace(1) [[IP11]], align 4
+; CHECK-NEXT: [[MP6:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 6
+; CHECK-NEXT: [[MV6:%.*]] = load float, ptr addrspace(4) [[MP6]], align 4
+; CHECK-NEXT: [[PROD6:%.*]] = fmul contract float [[IV6]], [[MV6]]
+; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC5]], [[PROD6]]
+; CHECK-NEXT: [[IP7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 10
+; CHECK-NEXT: [[IV7:%.*]] = load float, ptr addrspace(1) [[IP7]], align 4
+; CHECK-NEXT: [[MP7:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 7
+; CHECK-NEXT: [[MV7:%.*]] = load float, ptr addrspace(4) [[MP7]], align 4
+; CHECK-NEXT: [[PROD7:%.*]] = fmul contract float [[IV7]], [[MV7]]
+; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[PROD7]]
+; CHECK-NEXT: [[IP13:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
+; CHECK-NEXT: [[IV8:%.*]] = load float, ptr addrspace(1) [[IP13]], align 4
+; CHECK-NEXT: [[MP8:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 8
+; CHECK-NEXT: [[MV8:%.*]] = load float, ptr addrspace(4) [[MP8]], align 4
+; CHECK-NEXT: [[PROD8:%.*]] = fmul contract float [[IV8]], [[MV8]]
+; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[PROD8]]
; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12
; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4
+; CHECK-NEXT: [[MP9:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 9
+; CHECK-NEXT: [[MV9:%.*]] = load float, ptr addrspace(4) [[MP9]], align 4
+; CHECK-NEXT: [[PROD9:%.*]] = fmul contract float [[IV5]], [[MV9]]
+; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[PROD9]]
; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4
+; CHECK-NEXT: [[MP10:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 10
+; CHECK-NEXT: [[MV10:%.*]] = load float, ptr addrspace(4) [[MP10]], align 4
+; CHECK-NEXT: [[PROD10:%.*]] = fmul contract float [[IV14]], [[MV10]]
+; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[PROD10]]
; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17
+; CHECK-NEXT: [[IV11:%.*]] = load float, ptr addrspace(1) [[IP8]], align 4
+; CHECK-NEXT: [[MP11:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 11
+; CHECK-NEXT: [[MV11:%.*]] = load float, ptr addrspace(4) [[MP11]], align 4
+; CHECK-NEXT: [[PROD11:%.*]] = fmul contract float [[IV11]], [[MV11]]
+; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[PROD11]]
+; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
+; CHECK-NEXT: [[IV12:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4
+; CHECK-NEXT: [[MP12:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 12
+; CHECK-NEXT: [[MV12:%.*]] = load float, ptr addrspace(4) [[MP12]], align 4
+; CHECK-NEXT: [[PROD12:%.*]] = fmul contract float [[IV12]], [[MV12]]
+; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[PROD12]]
; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19
+; CHECK-NEXT: [[IV13:%.*]] = load float, ptr addrspace(1) [[IP10]], align 4
+; CHECK-NEXT: [[MP13:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 13
+; CHECK-NEXT: [[MV13:%.*]] = load float, ptr addrspace(4) [[MP13]], align 4
+; CHECK-NEXT: [[PROD13:%.*]] = fmul contract float [[IV13]], [[MV13]]
+; CHECK-NEXT: [[ACC14:%.*]] = fadd contract float [[ACC12]], [[PROD13]]
+; CHECK-NEXT: [[IP25:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20
+; CHECK-NEXT: [[IV25:%.*]] = load float, ptr addrspace(1) [[IP25]], align 4
+; CHECK-NEXT: [[MP14:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 14
+; CHECK-NEXT: [[MV14:%.*]] = load float, ptr addrspace(4) [[MP14]], align 4
+; CHECK-NEXT: [[PROD14:%.*]] = fmul contract float [[IV25]], [[MV14]]
+; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC14]], [[PROD14]]
; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[IP6]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV5]], i64 8
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV14]], i64 9
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP20]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP24]], [[TMP6]]
-; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
-; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[ACC14]]
-; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
-; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC1]], [[PROD15]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
-; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
-; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP21]], i64 4
-; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[TMP14]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP21]], i64 5
-; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[TMP15]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x float> [[TMP21]], i64 6
-; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC19]], [[TMP28]]
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <16 x float> [[TMP21]], i64 7
-; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[TMP29]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <16 x float> [[TMP21]], i64 8
-; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[TMP30]]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <16 x float> [[TMP21]], i64 9
-; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[TMP31]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <16 x float> [[TMP21]], i64 10
-; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[TMP32]]
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <16 x float> [[TMP21]], i64 11
-; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[TMP33]]
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <16 x float> [[TMP21]], i64 12
-; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[TMP34]]
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <16 x float> [[TMP21]], i64 13
-; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP35]]
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <16 x float> [[TMP21]], i64 14
+; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP12]], align 4
+; CHECK-NEXT: [[MP15:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 15
+; CHECK-NEXT: [[MV15:%.*]] = load float, ptr addrspace(4) [[MP15]], align 4
+; CHECK-NEXT: [[TMP36:%.*]] = fmul contract float [[IV15]], [[MV15]]
; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
+; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
+; CHECK-NEXT: [[IV16:%.*]] = load float, ptr addrspace(1) [[IP16]], align 4
+; CHECK-NEXT: [[MP17:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
+; CHECK-NEXT: [[MV16:%.*]] = load float, ptr addrspace(4) [[MP17]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = fmul contract float [[IV16]], [[MV16]]
; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
; CHECK-NEXT: [[IP17:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 26
+; CHECK-NEXT: [[IV17:%.*]] = load float, ptr addrspace(1) [[IP17]], align 4
; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 17
+; CHECK-NEXT: [[MV17:%.*]] = load float, ptr addrspace(4) [[MP16]], align 4
+; CHECK-NEXT: [[PROD17:%.*]] = fmul contract float [[IV17]], [[MV17]]
+; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC26]], [[PROD17]]
+; CHECK-NEXT: [[IP18:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 27
+; CHECK-NEXT: [[IV18:%.*]] = load float, ptr addrspace(1) [[IP18]], align 4
+; CHECK-NEXT: [[MP18:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 18
+; CHECK-NEXT: [[MV18:%.*]] = load float, ptr addrspace(4) [[MP18]], align 4
+; CHECK-NEXT: [[PROD18:%.*]] = fmul contract float [[IV18]], [[MV18]]
+; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[PROD18]]
; CHECK-NEXT: [[IP19:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 28
; CHECK-NEXT: [[IV19:%.*]] = load float, ptr addrspace(1) [[IP19]], align 4
+; CHECK-NEXT: [[MP19:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 19
+; CHECK-NEXT: [[MV19:%.*]] = load float, ptr addrspace(4) [[MP19]], align 4
+; CHECK-NEXT: [[PROD19:%.*]] = fmul contract float [[IV19]], [[MV19]]
+; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[PROD19]]
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 20
+; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4
+; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]]
+; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC19]], [[PROD20]]
; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33
-; CHECK-NEXT: [[TMP38:%.*]] = load <2 x float>, ptr addrspace(1) [[IP17]], align 4
-; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP21]], align 4
-; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = insertelement <8 x float> poison, float [[IV19]], i64 2
-; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP41]], float [[IV20]], i64 3
-; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <8 x float> [[TMP53]], <8 x float> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x float> [[TMP38]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <8 x float> [[TMP54]], <8 x float> [[TMP55]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
-; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]]
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x float> [[TMP44]], i64 1
-; CHECK-NEXT: [[ACC28:%.*]] = fadd contract float [[ACC27]], [[TMP46]]
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x float> [[TMP44]], i64 2
-; CHECK-NEXT: [[ACC29:%.*]] = fadd contract float [[ACC28]], [[TMP47]]
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <8 x float> [[TMP44]], i64 3
-; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC29]], [[TMP48]]
-; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x float> [[TMP44]], i64 4
+; CHECK-NEXT: [[IV21:%.*]] = load float, ptr addrspace(1) [[IP21]], align 4
+; CHECK-NEXT: [[MP21:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 21
+; CHECK-NEXT: [[MV21:%.*]] = load float, ptr addrspace(4) [[MP21]], align 4
+; CHECK-NEXT: [[TMP49:%.*]] = fmul contract float [[IV21]], [[MV21]]
; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC30]], [[TMP49]]
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x float> [[TMP44]], i64 5
+; CHECK-NEXT: [[IP22:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 34
+; CHECK-NEXT: [[IV22:%.*]] = load float, ptr addrspace(1) [[IP22]], align 4
+; CHECK-NEXT: [[MP22:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 22
+; CHECK-NEXT: [[MV22:%.*]] = load float, ptr addrspace(4) [[MP22]], align 4
+; CHECK-NEXT: [[TMP50:%.*]] = fmul contract float [[IV22]], [[MV22]]
; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP50]]
-; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6
+; CHECK-NEXT: [[IP23:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 35
+; CHECK-NEXT: [[IV23:%.*]] = load float, ptr addrspace(1) [[IP23]], align 4
+; CHECK-NEXT: [[MP23:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 23
+; CHECK-NEXT: [[MV23:%.*]] = load float, ptr addrspace(4) [[MP23]], align 4
+; CHECK-NEXT: [[TMP51:%.*]] = fmul contract float [[IV23]], [[MV23]]
; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]]
-; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7
+; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36
+; CHECK-NEXT: [[IV24:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4
+; CHECK-NEXT: [[MP24:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24
+; CHECK-NEXT: [[MV24:%.*]] = load float, ptr addrspace(4) [[MP24]], align 4
+; CHECK-NEXT: [[TMP52:%.*]] = fmul contract float [[IV24]], [[MV24]]
; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
; CHECK-NEXT: ret float [[ACC24]]
;
More information about the llvm-branch-commits
mailing list