[llvm] [AMDGPU] Sink a single-use fmul into the block of its fadd/fsub user (PR #215810)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 10:26:59 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/215810
>From 537acf349ba74ba7404fce6f21fcbcb8925c1384 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 30 Jul 2026 12:34:06 -0500
Subject: [PATCH 1/5] [AMDGPU] Sink a single-use fmul into the block of its
fadd/fsub user
The cost model prices an fmul as free when its only user is a
contractable fadd/fsub, on the assumption the pair is selected as one
fused instruction. That only holds when both sit in the same block, and
a loop-invariant fmul hoisted out of its user's loop is left behind and
never fused, while still priced as free.
The generic sinking pass handles the plain case but will not sink into a
loop, which is exactly where the fmul is stranded. Sink it back into the
user's block when the two would fuse, using the same condition the cost
model does so they cannot disagree. Only the operand that would actually
fuse is moved, and only when it has a single use so this stays a move
rather than a copy.
Derive that shared condition from instruction selection's own mad
legality check rather than re-deriving it here. On targets without a
16-bit mad the pair does not actually fuse, so the f16 fmul is no longer
priced free and is not sunk, which is where the f16 test changes come
from.
Assisted-by: Claude Code Opus 5
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 58 ++++--
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 7 +-
.../Analysis/CostModel/AMDGPU/fused_costs.ll | 117 ++++++-----
.../global-atomicrmw-fadd-wrong-subtarget.ll | 7 +-
.../AMDGPU/global_atomics_scan_fadd.ll | 44 ++--
.../AMDGPU/global_atomics_scan_fsub.ll | 44 ++--
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll | 195 ++++++++----------
.../AMDGPU/sink-fmul-fadd-contract-fast.ll | 2 +-
.../CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll | 166 ++++++++++-----
9 files changed, 356 insertions(+), 284 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 980e26082064f..e2f192a2c5bf8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -290,8 +290,6 @@ GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {
SIModeRegisterDefaults Mode(F, *ST);
HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
- HasFP64FP16Denormals =
- Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();
}
bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
@@ -524,6 +522,24 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
}
}
+bool GCNTTIImpl::canFuseFMulWithFAddSub(MVT::SimpleValueType SLT,
+ const Instruction *FMul,
+ const Instruction *FAddSub) const {
+ const int OPC = TLI->InstructionOpcodeToISD(FAddSub->getOpcode());
+ if (OPC != ISD::FADD && OPC != ISD::FSUB)
+ return false;
+
+ // The mad forms fuse exactly without fast-math flags but flush denormals.
+ const DenormalFPEnv FPEnv = FAddSub->getFunction()->getDenormalFPEnv();
+ if (TLI->isFMADLegal(MVT(SLT), FPEnv))
+ return true;
+
+ // Other types fuse only with contract or fast.
+ const TargetOptions &Options = TLI->getTargetMachine().Options;
+ return Options.AllowFPOpFusion == FPOpFusion::Fast ||
+ (FAddSub->hasAllowContract() && FMul->hasAllowContract());
+}
+
InstructionCost GCNTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
@@ -587,21 +603,9 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
// fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
// fused operation.
if (CxtI && CxtI->hasOneUse())
- if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin())) {
- const int OPC = TLI->InstructionOpcodeToISD(FAdd->getOpcode());
- if (OPC == ISD::FADD || OPC == ISD::FSUB) {
- if (ST->hasMadMacF32Insts() && SLT == MVT::f32 && !HasFP32Denormals)
- return TargetTransformInfo::TCC_Free;
- if (ST->has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)
- return TargetTransformInfo::TCC_Free;
-
- // Estimate all types may be fused with contract/unsafe flags
- const TargetOptions &Options = TLI->getTargetMachine().Options;
- if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
- (FAdd->hasAllowContract() && CxtI->hasAllowContract()))
- return TargetTransformInfo::TCC_Free;
- }
- }
+ if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin()))
+ if (canFuseFMulWithFAddSub(SLT, CxtI, FAdd))
+ return TargetTransformInfo::TCC_Free;
[[fallthrough]];
case ISD::FADD:
case ISD::FSUB:
@@ -1483,6 +1487,26 @@ bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const {
using namespace PatternMatch;
+ // The cost model prices this fmul as free assuming it fuses with its
+ // fadd/fsub user, which needs them in one block. Sink a stranded
+ // loop-invariant fmul back to the user when they would fuse. Single use only,
+ // so this stays a move.
+ if (I->getOpcode() == Instruction::FAdd ||
+ I->getOpcode() == Instruction::FSub) {
+ MVT::SimpleValueType SLT =
+ getTypeLegalizationCost(I->getType()).second.getScalarType().SimpleTy;
+ for (Use &Op : I->operands()) {
+ auto *FMul = dyn_cast<Instruction>(Op.get());
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse() || !canFuseFMulWithFAddSub(SLT, FMul, I))
+ continue;
+ // The fused operand. Sink it when it sits in another block, then stop.
+ if (FMul->getParent() != I->getParent())
+ Ops.push_back(&Op);
+ break;
+ }
+ }
+
for (auto &Op : I->operands()) {
// Ensure we are not already sinking this operand.
if (any_of(Ops, [&](Use *U) { return U->get() == Op.get(); }))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index df7b6d339e6c2..155dcca92f4ac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -71,7 +71,6 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
AMDGPUTTIImpl CommonTTI;
bool IsGraphics;
bool HasFP32Denormals;
- bool HasFP64FP16Denormals;
static constexpr bool InlinerVectorBonusPercent = 0;
static const FeatureBitset InlineFeatureIgnoreList;
@@ -103,6 +102,12 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
+ /// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
+ /// expected to fuse during instruction selection. \p SLT is the legalized
+ /// scalar type.
+ bool canFuseFMulWithFAddSub(MVT::SimpleValueType SLT, const Instruction *FMul,
+ const Instruction *FAddSub) const;
+
/// \returns true if V might be divergent even when all of its operands
/// are uniform.
bool isSourceOfDivergence(const Value *V) const;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
index 98773ba7292f4..858292b469fba 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
@@ -92,65 +92,65 @@ define void @fmul_fadd_f32() #0 {
}
define void @fmul_fadd_f16() #0 {
-; FUSED-LABEL: 'fmul_fadd_f16'
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; SLOWF32-LABEL: 'fmul_fadd_f16'
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX9SLOW-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; FASTF32-LABEL: 'fmul_fadd_f16'
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; FUSED-SIZE-LABEL: 'fmul_fadd_f16'
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+; SLOWF32-SIZE-LABEL: 'fmul_fadd_f16'
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
-; GFX9SLOW-SIZE-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+; FASTF32-SIZE-LABEL: 'fmul_fadd_f16'
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
%f16 = fmul half undef, undef
%f16add = fadd half %f16, undef
@@ -255,3 +255,8 @@ define void @fmul_fadd_f64() #0 {
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FUSED: {{.*}}
+; FUSED-SIZE: {{.*}}
+; GFX9SLOW: {{.*}}
+; GFX9SLOW-SIZE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
index a1da18969552a..d4870b7ca8a95 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
@@ -14,18 +14,17 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrsp
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: s_mov_b64 s[6:7], 0
-; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_load_dword s8, s[4:5], 0x0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s8
; GCN-NEXT: .LBB0_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_add_f32_e32 v4, v5, v2
+; GCN-NEXT: v_mad_f32 v4, 4.0, v2, v5
; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index d307ffaff5c63..8672b6eab90de 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -27,18 +27,17 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
@@ -63,17 +62,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX9-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -96,16 +94,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1064-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -128,15 +125,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1032-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
@@ -198,18 +194,17 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
@@ -234,17 +229,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX9-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -267,16 +261,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1064-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -299,15 +292,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1032-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 327928e152f95..47891774dfabb 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -27,18 +27,17 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
@@ -63,17 +62,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX9-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -96,16 +94,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1064-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -128,15 +125,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1032-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
@@ -228,18 +224,17 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
@@ -264,17 +259,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX9-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -297,16 +291,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1064-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -329,15 +322,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1032-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
index c01e3b436a3f1..c49906c0818dd 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -25,10 +25,11 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB0_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -44,7 +45,7 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -53,7 +54,7 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f32 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -63,14 +64,14 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_f32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_fmac_f32_e32 v0, v3, v1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -116,10 +117,11 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fsub_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB1_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -135,7 +137,7 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-LABEL: fsub_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -144,7 +146,7 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f32 v0, -v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -154,14 +156,14 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX10-LABEL: fsub_in_loop_f32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB1_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_fma_f32 v0, -v3, v1, v0
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -207,7 +209,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -216,7 +218,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_fma_f16 v0, v3, v1, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB2_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -226,7 +228,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -235,7 +237,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f16 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB2_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -245,14 +247,14 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_f16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB2_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -300,14 +302,13 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f64:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_mov_b32_e32 v6, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB3_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX8-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -315,19 +316,20 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
; GFX8-NEXT: ; %bb.2: ; %exit
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fma_in_loop_f64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB3_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -335,19 +337,20 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX9-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-NEXT: ; %bb.2: ; %exit
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fma_in_loop_f64:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_mov_b32_e32 v6, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB3_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX10-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX10-NEXT: s_add_i32 s5, s5, 1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -355,6 +358,8 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
; GFX10-NEXT: ; %bb.2: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
%mul = fmul contract double %a, %b
@@ -395,7 +400,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX8-LABEL: mad_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -404,7 +409,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_mac_f32_e32 v0, v3, v1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -414,7 +419,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX9-LABEL: mad_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -423,7 +428,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mac_f32_e32 v0, v3, v1
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -658,7 +663,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX8-LABEL: mad_in_loop_f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -667,7 +672,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -677,7 +682,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX9-LABEL: mad_in_loop_f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -686,7 +691,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -732,20 +737,20 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX8-LABEL: no_fma_in_loop_v2f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB8_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_mac_f16_sdwa v4, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -755,10 +760,11 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX9-LABEL: no_fma_in_loop_v2f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX9-NEXT: .LBB8_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -774,10 +780,11 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX10-LABEL: no_fma_in_loop_v2f16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX10-NEXT: .LBB8_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
@@ -809,30 +816,32 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX8-LABEL: fma_in_loop_v2f16_denormals:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB9_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_fma_f16 v4, v6, v5, v4
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT: v_fma_f16 v3, v0, v1, v3
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %exit
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fma_in_loop_v2f16_denormals:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -841,7 +850,7 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT: v_pk_fma_f16 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -851,14 +860,14 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX10-LABEL: fma_in_loop_v2f16_denormals:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB9_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX10-NEXT: v_pk_fma_f16 v0, v3, v1, v0
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -887,31 +896,23 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_bf16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: .LBB10_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX8-NEXT: v_fma_f32 v0, v3, v1, v0
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -924,31 +925,24 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_bf16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s6, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s6
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_mov_b32 s7, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: s_movk_i32 s7, 0x7fff
; GFX9-NEXT: .LBB10_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX9-NEXT: v_add3_u32 v3, v3, v0, s6
-; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT: v_fma_f32 v0, v3, v1, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s7
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: s_add_i32 s7, s7, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s7, v2
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -960,30 +954,23 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_bf16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: .p2align 6
; GFX10-NEXT: .LBB10_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_fmac_f32_e32 v0, v3, v1
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
index a8a4d12a502e3..2fd2600197aa1 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
@@ -7,9 +7,9 @@ define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fadd_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
index b548b99663f9e..602d9ebd0dadc 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
@@ -8,9 +8,9 @@ define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fadd_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -34,9 +34,9 @@ define float @sink_fmul_fsub_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fsub_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -61,9 +61,9 @@ define float @sink_fmul_fsub_rev_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fsub_rev_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[MUL]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -87,9 +87,9 @@ define half @sink_fmul_fadd_f16(i1 %cond, half %a, half %b, half %c) {
; CHECK-LABEL: define half @sink_fmul_fadd_f16(
; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract half [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract half [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract half [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -113,9 +113,9 @@ define double @sink_fmul_fadd_f64(i1 %cond, double %a, double %b, double %c) {
; CHECK-LABEL: define double @sink_fmul_fadd_f64(
; CHECK-SAME: i1 [[COND:%.*]], double [[A:%.*]], double [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract double [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract double [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract double [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -139,9 +139,9 @@ define <2 x float> @sink_fmul_fadd_v2f32(i1 %cond, <2 x float> %a, <2 x float> %
; CHECK-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
; CHECK-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -166,10 +166,10 @@ define float @sink_only_fusable_fmul(i1 %cond, float %a, float %b, float %c, flo
; CHECK-LABEL: define float @sink_only_fusable_fmul(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -221,17 +221,41 @@ exit:
; v_mad_f32 is exact, so with denormals flushed no fast-math flags are needed.
define float @sink_fmul_fadd_f32_no_denormals(i1 %cond, float %a, float %b, float %c) #0 {
-; CHECK-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul = fmul float %a, %b
@@ -248,17 +272,41 @@ exit:
; Same for v_mad_f16.
define half @sink_fmul_fadd_f16_no_denormals(i1 %cond, half %a, half %b, half %c) #0 {
-; CHECK-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul half [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd half [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret half [[R]]
+; GFX8-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX8-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul half [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd half [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret half [[R]]
+;
+; GFX9-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX9-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul half [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd half [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret half [[R]]
+;
+; GFX10-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX10-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul half [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd half [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret half [[R]]
;
entry:
%mul = fmul half %a, %b
@@ -428,11 +476,11 @@ define float @sink_fmul_into_loop(float %a, float %b, i32 %n) {
; CHECK-LABEL: define float @sink_fmul_into_loop(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD]] = fadd contract float [[ACC]], [[MUL]]
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
@@ -459,11 +507,11 @@ exit:
; A legal v2f16 has no packed mad, and v_pk_fma_f16 needs denormals enabled.
define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -485,17 +533,41 @@ exit:
; Same without the contract flags.
define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
-; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x half> [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd <2 x half> [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT: ret <2 x half> [[R]]
+; GFX8-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX8-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd <2 x half> [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX8-NEXT: ret <2 x half> [[R]]
+;
+; GFX9-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX9-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd <2 x half> [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX9-NEXT: ret <2 x half> [[R]]
+;
+; GFX10-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX10-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd <2 x half> [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX10-NEXT: ret <2 x half> [[R]]
;
entry:
%mul = fmul <2 x half> %a, %b
@@ -515,9 +587,9 @@ define <2 x half> @sink_fmul_fadd_v2f16_denormals(i1 %cond, <2 x half> %a, <2 x
; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_denormals(
; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -542,9 +614,9 @@ define bfloat @sink_fmul_fadd_bf16(i1 %cond, bfloat %a, bfloat %b, bfloat %c) {
; CHECK-LABEL: define bfloat @sink_fmul_fadd_bf16(
; CHECK-SAME: i1 [[COND:%.*]], bfloat [[A:%.*]], bfloat [[B:%.*]], bfloat [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract bfloat [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract bfloat [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract bfloat [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -565,7 +637,3 @@ exit:
}
attributes #0 = { denormal_fpenv(preservesign) }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX8: {{.*}}
-; GFX9: {{.*}}
>From bb21a8173407f6d15ec9ec8cef182dd1d8a1bf71 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 20 Aug 2026 01:23:19 +0200
Subject: [PATCH 2/5] apply comments
---
.../Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 16 ++++++----------
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 6 +++---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 ++++++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 3 +++
4 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index e2f192a2c5bf8..a4e52d4932872 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -522,16 +522,14 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
}
}
-bool GCNTTIImpl::canFuseFMulWithFAddSub(MVT::SimpleValueType SLT,
- const Instruction *FMul,
+bool GCNTTIImpl::canFuseFMulWithFAddSub(Type *Ty, const Instruction *FMul,
const Instruction *FAddSub) const {
- const int OPC = TLI->InstructionOpcodeToISD(FAddSub->getOpcode());
- if (OPC != ISD::FADD && OPC != ISD::FSUB)
+ const unsigned Opc = FAddSub->getOpcode();
+ if (Opc != Instruction::FAdd && Opc != Instruction::FSub)
return false;
// The mad forms fuse exactly without fast-math flags but flush denormals.
- const DenormalFPEnv FPEnv = FAddSub->getFunction()->getDenormalFPEnv();
- if (TLI->isFMADLegal(MVT(SLT), FPEnv))
+ if (TLI->isFMADLegal(*FAddSub->getFunction(), Ty))
return true;
// Other types fuse only with contract or fast.
@@ -604,7 +602,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
// fused operation.
if (CxtI && CxtI->hasOneUse())
if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin()))
- if (canFuseFMulWithFAddSub(SLT, CxtI, FAdd))
+ if (canFuseFMulWithFAddSub(Ty, CxtI, FAdd))
return TargetTransformInfo::TCC_Free;
[[fallthrough]];
case ISD::FADD:
@@ -1493,12 +1491,10 @@ bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
// so this stays a move.
if (I->getOpcode() == Instruction::FAdd ||
I->getOpcode() == Instruction::FSub) {
- MVT::SimpleValueType SLT =
- getTypeLegalizationCost(I->getType()).second.getScalarType().SimpleTy;
for (Use &Op : I->operands()) {
auto *FMul = dyn_cast<Instruction>(Op.get());
if (!FMul || FMul->getOpcode() != Instruction::FMul ||
- !FMul->hasOneUse() || !canFuseFMulWithFAddSub(SLT, FMul, I))
+ !FMul->hasOneUse() || !canFuseFMulWithFAddSub(I->getType(), FMul, I))
continue;
// The fused operand. Sink it when it sits in another block, then stop.
if (FMul->getParent() != I->getParent())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 155dcca92f4ac..646e248021bc9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -103,9 +103,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
/// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
- /// expected to fuse during instruction selection. \p SLT is the legalized
- /// scalar type.
- bool canFuseFMulWithFAddSub(MVT::SimpleValueType SLT, const Instruction *FMul,
+ /// expected to fuse during instruction selection. \p Ty is the type the
+ /// fused operation runs on.
+ bool canFuseFMulWithFAddSub(Type *Ty, const Instruction *FMul,
const Instruction *FAddSub) const;
/// \returns true if V might be divergent even when all of its operands
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f9943c6208656..0eb9a4e0f15d9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7546,6 +7546,12 @@ bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
getDenormalFPEnv(DAG.getMachineFunction()));
}
+bool SITargetLowering::isFMADLegal(const Function &F, Type *Ty) const {
+ return isFMADLegal(getValueType(F.getDataLayout(), Ty->getScalarType(),
+ /*AllowUnknown=*/true),
+ F.getDenormalFPEnv());
+}
+
//===----------------------------------------------------------------------===//
// Custom DAG Lowering Operations
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 3f5b03d631c48..413482f290159 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -513,6 +513,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// \p VT is used as written, so a vector type reports false.
bool isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const;
+ /// \p Ty is taken by its scalar type, so a vector type asks about a lane.
+ bool isFMADLegal(const Function &F, Type *Ty) const;
+
bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
>From d1203dcd74f5f8a246eb83c3ad805032479192c8 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 20 Aug 2026 13:13:31 +0200
Subject: [PATCH 3/5] update test after merge
---
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
index c49906c0818dd..7f0491c2309c5 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -819,18 +819,18 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: .LBB9_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT: v_fma_f16 v4, v6, v5, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX8-NEXT: v_fma_f16 v6, v5, v4, v6
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_fma_f16 v3, v0, v1, v3
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v6
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %exit
>From 99e64b28aa1b7f61f3584bf4f5c84b391553b8ff Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 21 Aug 2026 15:25:19 +0200
Subject: [PATCH 4/5] Require an fma to be faster before pricing an fmul as
fused
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 +-
.../Analysis/CostModel/AMDGPU/fused_costs.ll | 92 +++--
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll | 9 +-
.../CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll | 338 +++++++++++++-----
4 files changed, 328 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a4e52d4932872..b93a3405764ca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -529,12 +529,16 @@ bool GCNTTIImpl::canFuseFMulWithFAddSub(Type *Ty, const Instruction *FMul,
return false;
// The mad forms fuse exactly without fast-math flags but flush denormals.
- if (TLI->isFMADLegal(*FAddSub->getFunction(), Ty))
- return true;
+ // An fma forms only when it is not slower than the separate operations.
+ const Function &F = *FAddSub->getFunction();
+ const bool HasFMAD = TLI->isFMADLegal(F, Ty);
+ const bool HasFMA = TLI->isFMAFasterThanFMulAndFAdd(F, Ty);
+ if (!HasFMAD && !HasFMA)
+ return false;
- // Other types fuse only with contract or fast.
+ // Without a mad the pair fuses only with contract or fast.
const TargetOptions &Options = TLI->getTargetMachine().Options;
- return Options.AllowFPOpFusion == FPOpFusion::Fast ||
+ return HasFMAD || Options.AllowFPOpFusion == FPOpFusion::Fast ||
(FAddSub->hasAllowContract() && FMul->hasAllowContract());
}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
index 858292b469fba..772c2f76dd711 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
@@ -2,12 +2,12 @@
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64,FUSED,SLOWF32 %s
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=ieee -denormal-fp-math=ieee -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64,FASTF32,GFX9SLOW %s
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=ieee -denormal-fp-math=ieee -fp-contract=fast < %s | FileCheck -check-prefixes=FUSED,SLOWF32,GFX9FAST %s
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu10.30-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64,FUSED,FASTF32 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu10.30-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64,FUSED,FASTF32,GFX1030 %s
; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64-SIZE,FUSED-SIZE,SLOWF32-SIZE %s
; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=ieee -denormal-fp-math=ieee -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64-SIZE,FASTF32-SIZE,GFX9SLOW-SIZE %s
; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-unknown-amdhsa -denormal-fp-math-f32=ieee -denormal-fp-math=ieee -fp-contract=fast < %s | FileCheck -check-prefixes=FUSED-SIZE,SLOWF32-SIZE,GFX9FAST-SIZE %s
-; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu10.30-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64-SIZE,FUSED-SIZE,FASTF32-SIZE %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu10.30-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64-SIZE,FUSED-SIZE,FASTF32-SIZE,GFX1030-SIZE %s
; END.
define void @fmul_fadd_f32() #0 {
@@ -107,20 +107,35 @@ define void @fmul_fadd_f16() #0 {
; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
; SLOWF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; FASTF32-LABEL: 'fmul_fadd_f16'
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX9SLOW-LABEL: 'fmul_fadd_f16'
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX1030-LABEL: 'fmul_fadd_f16'
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; SLOWF32-SIZE-LABEL: 'fmul_fadd_f16'
; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
@@ -137,20 +152,35 @@ define void @fmul_fadd_f16() #0 {
; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
-; FASTF32-SIZE-LABEL: 'fmul_fadd_f16'
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+; GFX9SLOW-SIZE-LABEL: 'fmul_fadd_f16'
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; GFX1030-SIZE-LABEL: 'fmul_fadd_f16'
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
%f16 = fmul half undef, undef
%f16add = fadd half %f16, undef
@@ -258,5 +288,3 @@ attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; FUSED: {{.*}}
; FUSED-SIZE: {{.*}}
-; GFX9SLOW: {{.*}}
-; GFX9SLOW-SIZE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
index 7f0491c2309c5..ccaffb0696df1 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -25,11 +25,10 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB0_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -117,11 +116,10 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fsub_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB1_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -780,11 +778,10 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX10-LABEL: no_fma_in_loop_v2f16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX10-NEXT: .LBB8_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
index 602d9ebd0dadc..55347a00cdaad 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
@@ -5,17 +5,41 @@
define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
-; CHECK-LABEL: define float @sink_fmul_fadd_f32(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
-; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_fmul_fadd_f32(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_fmul_fadd_f32(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd contract float [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_fmul_fadd_f32(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX10-NEXT: [[ADD:%.*]] = fadd contract float [[TMP0]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul = fmul contract float %a, %b
@@ -31,17 +55,41 @@ exit:
}
define float @sink_fmul_fsub_f32(i1 %cond, float %a, float %b, float %c) {
-; CHECK-LABEL: define float @sink_fmul_fsub_f32(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_fmul_fsub_f32(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[SUB:%.*]] = fsub contract float [[MUL]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_fmul_fsub_f32(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX9-NEXT: [[SUB:%.*]] = fsub contract float [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_fmul_fsub_f32(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX10-NEXT: [[SUB:%.*]] = fsub contract float [[TMP0]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul = fmul contract float %a, %b
@@ -58,17 +106,41 @@ exit:
; The fmul may be the second operand of the fsub as well.
define float @sink_fmul_fsub_rev_f32(i1 %cond, float %a, float %b, float %c) {
-; CHECK-LABEL: define float @sink_fmul_fsub_rev_f32(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[MUL]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_fmul_fsub_rev_f32(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[MUL]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_fmul_fsub_rev_f32(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX9-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[TMP0]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_fmul_fsub_rev_f32(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX10-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[TMP0]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul = fmul contract float %a, %b
@@ -136,17 +208,41 @@ exit:
}
define <2 x float> @sink_fmul_fadd_v2f32(i1 %cond, <2 x float> %a, <2 x float> %b, <2 x float> %c) {
-; CHECK-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
-; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi <2 x float> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT: ret <2 x float> [[R]]
+; GFX8-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
+; GFX8-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[MUL]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi <2 x float> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX8-NEXT: ret <2 x float> [[R]]
+;
+; GFX9-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
+; GFX9-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract <2 x float> [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi <2 x float> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX9-NEXT: ret <2 x float> [[R]]
+;
+; GFX10-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
+; GFX10-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract <2 x float> [[A]], [[B]]
+; GFX10-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[TMP0]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi <2 x float> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX10-NEXT: ret <2 x float> [[R]]
;
entry:
%mul = fmul contract <2 x float> %a, %b
@@ -163,18 +259,44 @@ exit:
; Only the operand DAGCombiner picks is a candidate.
define float @sink_only_fusable_fmul(i1 %cond, float %a, float %b, float %c, float %d) {
-; CHECK-LABEL: define float @sink_only_fusable_fmul(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
-; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_only_fusable_fmul(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX8-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_only_fusable_fmul(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd contract float [[TMP0]], [[MUL1]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_only_fusable_fmul(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX10-NEXT: [[ADD:%.*]] = fadd contract float [[TMP0]], [[MUL1]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul0 = fmul contract float %a, %b
@@ -473,20 +595,50 @@ exit:
; The motivating shape, a loop-invariant fmul hoisted by LICM.
define float @sink_fmul_into_loop(float %a, float %b, i32 %n) {
-; CHECK-LABEL: define float @sink_fmul_into_loop(
-; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
-; CHECK-NEXT: [[ADD]] = fadd contract float [[ACC]], [[MUL]]
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret float [[ADD]]
+; GFX8-LABEL: define float @sink_fmul_into_loop(
+; GFX8-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; GFX8-NEXT: br label %[[LOOP:.*]]
+; GFX8: [[LOOP]]:
+; GFX8-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; GFX8-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; GFX8-NEXT: [[ADD]] = fadd contract float [[ACC]], [[MUL]]
+; GFX8-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GFX8-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GFX8-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: ret float [[ADD]]
+;
+; GFX9-LABEL: define float @sink_fmul_into_loop(
+; GFX9-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br label %[[LOOP:.*]]
+; GFX9: [[LOOP]]:
+; GFX9-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; GFX9-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX9-NEXT: [[ADD]] = fadd contract float [[ACC]], [[TMP0]]
+; GFX9-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GFX9-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GFX9-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: ret float [[ADD]]
+;
+; GFX10-LABEL: define float @sink_fmul_into_loop(
+; GFX10-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: br label %[[LOOP:.*]]
+; GFX10: [[LOOP]]:
+; GFX10-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; GFX10-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; GFX10-NEXT: [[TMP0:%.*]] = fmul contract float [[A]], [[B]]
+; GFX10-NEXT: [[ADD]] = fadd contract float [[ACC]], [[TMP0]]
+; GFX10-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GFX10-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GFX10-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: ret float [[ADD]]
;
entry:
%mul = fmul contract float %a, %b
@@ -506,17 +658,41 @@ exit:
; A legal v2f16 has no packed mad, and v_pk_fma_f16 needs denormals enabled.
define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
-; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
-; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT: ret <2 x half> [[R]]
+; GFX8-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
+; GFX8-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX8-NEXT: ret <2 x half> [[R]]
+;
+; GFX9-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
+; GFX9-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX9-NEXT: ret <2 x half> [[R]]
+;
+; GFX10-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
+; GFX10-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX10-NEXT: ret <2 x half> [[R]]
;
entry:
%mul = fmul contract <2 x half> %a, %b
>From bdbb5c219337840395d1bf472915278a1f734509 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 21 Aug 2026 19:22:24 +0200
Subject: [PATCH 5/5] undef
---
.../Analysis/CostModel/AMDGPU/fused_costs.ll | 414 +++++++++---------
1 file changed, 207 insertions(+), 207 deletions(-)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
index 772c2f76dd711..c1f5a6b18bdd6 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
@@ -10,276 +10,276 @@
; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu10.30-unknown-amdhsa -denormal-fp-math-f32=preserve-sign -denormal-fp-math=preserve-sign -fp-contract=on < %s | FileCheck -check-prefixes=SLOWF64-SIZE,FUSED-SIZE,FASTF32-SIZE,GFX1030-SIZE %s
; END.
-define void @fmul_fadd_f32() #0 {
+define void @fmul_fadd_f32(float %a, float %b, float %c, <2 x float> %va, <2 x float> %vb, <2 x float> %vc) #0 {
; SLOWF32-LABEL: 'fmul_fadd_f32'
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32 = fmul float undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32 = fmul <2 x float> undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32_2 = fmul float undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32_2 = fmul <2 x float> undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32 = fmul float %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32 = fmul <2 x float> %va, %vb
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, %vc
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32_2 = fmul float %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32_2 = fmul <2 x float> %va, %vb
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, %vc
; SLOWF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; FASTF32-LABEL: 'fmul_fadd_f32'
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32 = fmul float undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fmul <2 x float> undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32_2 = fmul float undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32_2 = fmul <2 x float> undef, undef
-; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32 = fmul float %a, %b
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, %c
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float %a, %b
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, %c
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fmul <2 x float> %va, %vb
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, %vc
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32_2 = fmul float %a, %b
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, %c
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float %a, %b
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, %c
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32_2 = fmul <2 x float> %va, %vb
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, %vc
; FASTF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; SLOWF32-SIZE-LABEL: 'fmul_fadd_f32'
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32 = fmul float undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32 = fmul <2 x float> undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32_2 = fmul float undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32_2 = fmul <2 x float> undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32 = fmul float %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32 = fmul <2 x float> %va, %vb
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, %vc
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32_2 = fmul float %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f32_2 = fmul <2 x float> %va, %vb
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, %vc
; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; FASTF32-SIZE-LABEL: 'fmul_fadd_f32'
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32 = fmul float undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fmul <2 x float> undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32_2 = fmul float undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32_2 = fmul <2 x float> undef, undef
-; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32 = fmul float %a, %b
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32add = fadd float %f32, %c
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c = fmul contract float %a, %b
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32cadd = fadd contract float %f32c, %c
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fmul <2 x float> %va, %vb
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32add = fadd <2 x float> %v2f32, %vc
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32_2 = fmul float %a, %b
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32sub = fsub float %f32_2, %c
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f32c_2 = fmul contract float %a, %b
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f32csub = fsub contract float %f32c_2, %c
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32_2 = fmul <2 x float> %va, %vb
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32sub = fsub <2 x float> %v2f32_2, %vc
; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
- %f32 = fmul float undef, undef
- %f32add = fadd float %f32, undef
+ %f32 = fmul float %a, %b
+ %f32add = fadd float %f32, %c
- %f32c = fmul contract float undef, undef
- %f32cadd = fadd contract float %f32c, undef
+ %f32c = fmul contract float %a, %b
+ %f32cadd = fadd contract float %f32c, %c
- %v2f32 = fmul <2 x float> undef, undef
- %v2f32add = fadd <2 x float> %v2f32, undef
+ %v2f32 = fmul <2 x float> %va, %vb
+ %v2f32add = fadd <2 x float> %v2f32, %vc
- %f32_2 = fmul float undef, undef
- %f32sub = fsub float %f32_2, undef
+ %f32_2 = fmul float %a, %b
+ %f32sub = fsub float %f32_2, %c
- %f32c_2 = fmul contract float undef, undef
- %f32csub = fsub contract float %f32c_2, undef
+ %f32c_2 = fmul contract float %a, %b
+ %f32csub = fsub contract float %f32c_2, %c
- %v2f32_2 = fmul <2 x float> undef, undef
- %v2f32sub = fsub <2 x float> %v2f32_2, undef
+ %v2f32_2 = fmul <2 x float> %va, %vb
+ %v2f32sub = fsub <2 x float> %v2f32_2, %vc
ret void
}
-define void @fmul_fadd_f16() #0 {
+define void @fmul_fadd_f16(half %a, half %b, half %c, <2 x half> %va, <2 x half> %vb, <2 x half> %vc) #0 {
; SLOWF32-LABEL: 'fmul_fadd_f16'
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half %a, %b
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; SLOWF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX9SLOW-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half %a, %b
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half %a, %b
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half %a, %b
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half %a, %b
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX1030-LABEL: 'fmul_fadd_f16'
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half %a, %b
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half %a, %b
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half %a, %b
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half %a, %b
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; GFX1030-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; GFX1030-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; SLOWF32-SIZE-LABEL: 'fmul_fadd_f16'
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half %a, %b
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; GFX9SLOW-SIZE-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half %a, %b
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half %a, %b
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half %a, %b
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half %a, %b
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; GFX1030-SIZE-LABEL: 'fmul_fadd_f16'
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half %a, %b
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, %c
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c = fmul contract half %a, %b
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, %c
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> %va, %vb
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, %vc
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half %a, %b
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, %c
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16c_2 = fmul contract half %a, %b
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, %c
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> %va, %vb
+; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, %vc
; GFX1030-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
- %f16 = fmul half undef, undef
- %f16add = fadd half %f16, undef
+ %f16 = fmul half %a, %b
+ %f16add = fadd half %f16, %c
- %f16c = fmul contract half undef, undef
- %f15cadd = fadd contract half %f16c, undef
+ %f16c = fmul contract half %a, %b
+ %f15cadd = fadd contract half %f16c, %c
- %v2f16 = fmul <2 x half> undef, undef
- %v2f16add = fadd <2 x half> %v2f16, undef
+ %v2f16 = fmul <2 x half> %va, %vb
+ %v2f16add = fadd <2 x half> %v2f16, %vc
- %f16_2 = fmul half undef, undef
- %f16sub = fsub half %f16_2, undef
+ %f16_2 = fmul half %a, %b
+ %f16sub = fsub half %f16_2, %c
- %f16c_2 = fmul contract half undef, undef
- %f15csub = fsub contract half %f16c_2, undef
+ %f16c_2 = fmul contract half %a, %b
+ %f15csub = fsub contract half %f16c_2, %c
- %v2f16_2 = fmul <2 x half> undef, undef
- %v2f16sub = fsub <2 x half> %v2f16_2, undef
+ %v2f16_2 = fmul <2 x half> %va, %vb
+ %v2f16sub = fsub <2 x half> %v2f16_2, %vc
ret void
}
-define void @fmul_fadd_f64() #0 {
+define void @fmul_fadd_f64(double %a, double %b, double %c, <2 x double> %va, <2 x double> %vb, <2 x double> %vc) #0 {
; SLOWF64-LABEL: 'fmul_fadd_f64'
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64 = fmul double undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64add = fadd double %f64, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64cadd = fadd contract double %f64c, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = fmul <2 x double> undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64add = fadd <2 x double> %v2f64, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64_2 = fmul double undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64sub = fsub double %f64_2, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64csub = fsub contract double %f64c_2, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64_2 = fmul <2 x double> undef, undef
-; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, undef
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64 = fmul double %a, %b
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64add = fadd double %f64, %c
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double %a, %b
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64cadd = fadd contract double %f64c, %c
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = fmul <2 x double> %va, %vb
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64add = fadd <2 x double> %v2f64, %vc
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64_2 = fmul double %a, %b
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64sub = fsub double %f64_2, %c
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double %a, %b
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64csub = fsub contract double %f64c_2, %c
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64_2 = fmul <2 x double> %va, %vb
+; SLOWF64-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, %vc
; SLOWF64-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX9FAST-LABEL: 'fmul_fadd_f64'
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64 = fmul double undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64add = fadd double %f64, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64cadd = fadd contract double %f64c, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64 = fmul <2 x double> undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64add = fadd <2 x double> %v2f64, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64_2 = fmul double undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64sub = fsub double %f64_2, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64csub = fsub contract double %f64c_2, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64_2 = fmul <2 x double> undef, undef
-; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, undef
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64 = fmul double %a, %b
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64add = fadd double %f64, %c
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double %a, %b
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64cadd = fadd contract double %f64c, %c
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64 = fmul <2 x double> %va, %vb
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64add = fadd <2 x double> %v2f64, %vc
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64_2 = fmul double %a, %b
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64sub = fsub double %f64_2, %c
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double %a, %b
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %f64csub = fsub contract double %f64c_2, %c
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64_2 = fmul <2 x double> %va, %vb
+; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, %vc
; GFX9FAST-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; SLOWF64-SIZE-LABEL: 'fmul_fadd_f64'
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64 = fmul double undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64add = fadd double %f64, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64cadd = fadd contract double %f64c, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64 = fmul <2 x double> undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64add = fadd <2 x double> %v2f64, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64_2 = fmul double undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64sub = fsub double %f64_2, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64csub = fsub contract double %f64c_2, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64_2 = fmul <2 x double> undef, undef
-; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, undef
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64 = fmul double %a, %b
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64add = fadd double %f64, %c
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double %a, %b
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64cadd = fadd contract double %f64c, %c
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64 = fmul <2 x double> %va, %vb
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64add = fadd <2 x double> %v2f64, %vc
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64_2 = fmul double %a, %b
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64sub = fsub double %f64_2, %c
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double %a, %b
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64csub = fsub contract double %f64c_2, %c
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64_2 = fmul <2 x double> %va, %vb
+; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, %vc
; SLOWF64-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; GFX9FAST-SIZE-LABEL: 'fmul_fadd_f64'
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64 = fmul double undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64add = fadd double %f64, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64cadd = fadd contract double %f64c, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64 = fmul <2 x double> undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64add = fadd <2 x double> %v2f64, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64_2 = fmul double undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64sub = fsub double %f64_2, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64csub = fsub contract double %f64c_2, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64_2 = fmul <2 x double> undef, undef
-; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, undef
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64 = fmul double %a, %b
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64add = fadd double %f64, %c
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c = fmul contract double %a, %b
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64cadd = fadd contract double %f64c, %c
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64 = fmul <2 x double> %va, %vb
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64add = fadd <2 x double> %v2f64, %vc
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64_2 = fmul double %a, %b
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64sub = fsub double %f64_2, %c
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f64c_2 = fmul contract double %a, %b
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %f64csub = fsub contract double %f64c_2, %c
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f64_2 = fmul <2 x double> %va, %vb
+; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2f64sub = fsub <2 x double> %v2f64_2, %vc
; GFX9FAST-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
- %f64 = fmul double undef, undef
- %f64add = fadd double %f64, undef
+ %f64 = fmul double %a, %b
+ %f64add = fadd double %f64, %c
- %f64c = fmul contract double undef, undef
- %f64cadd = fadd contract double %f64c, undef
+ %f64c = fmul contract double %a, %b
+ %f64cadd = fadd contract double %f64c, %c
- %v2f64 = fmul <2 x double> undef, undef
- %v2f64add = fadd <2 x double> %v2f64, undef
+ %v2f64 = fmul <2 x double> %va, %vb
+ %v2f64add = fadd <2 x double> %v2f64, %vc
- %f64_2 = fmul double undef, undef
- %f64sub = fsub double %f64_2, undef
+ %f64_2 = fmul double %a, %b
+ %f64sub = fsub double %f64_2, %c
- %f64c_2 = fmul contract double undef, undef
- %f64csub = fsub contract double %f64c_2, undef
+ %f64c_2 = fmul contract double %a, %b
+ %f64csub = fsub contract double %f64c_2, %c
- %v2f64_2 = fmul <2 x double> undef, undef
- %v2f64sub = fsub <2 x double> %v2f64_2, undef
+ %v2f64_2 = fmul <2 x double> %va, %vb
+ %v2f64sub = fsub <2 x double> %v2f64_2, %vc
ret void
}
More information about the llvm-commits
mailing list