[llvm] [AMDGPU] Sink a single-use fmul into the block of its fadd/fsub user (PR #215810)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 02:57:30 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/215810
>From 537acf349ba74ba7404fce6f21fcbcb8925c1384 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Thu, 30 Jul 2026 12:34:06 -0500
Subject: [PATCH] [AMDGPU] Sink a single-use fmul into the block of its
fadd/fsub user
The cost model prices an fmul as free when its only user is a
contractable fadd/fsub, on the assumption the pair is selected as one
fused instruction. That only holds when both sit in the same block, and
a loop-invariant fmul hoisted out of its user's loop is left behind and
never fused, while still priced as free.
The generic sinking pass handles the plain case but will not sink into a
loop, which is exactly where the fmul is stranded. Sink it back into the
user's block when the two would fuse, using the same condition the cost
model does so they cannot disagree. Only the operand that would actually
fuse is moved, and only when it has a single use so this stays a move
rather than a copy.
Derive that shared condition from instruction selection's own mad
legality check rather than re-deriving it here. On targets without a
16-bit mad the pair does not actually fuse, so the f16 fmul is no longer
priced free and is not sunk, which is where the f16 test changes come
from.
Assisted-by: Claude Code Opus 5
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 58 ++++--
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 7 +-
.../Analysis/CostModel/AMDGPU/fused_costs.ll | 117 ++++++-----
.../global-atomicrmw-fadd-wrong-subtarget.ll | 7 +-
.../AMDGPU/global_atomics_scan_fadd.ll | 44 ++--
.../AMDGPU/global_atomics_scan_fsub.ll | 44 ++--
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll | 195 ++++++++----------
.../AMDGPU/sink-fmul-fadd-contract-fast.ll | 2 +-
.../CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll | 166 ++++++++++-----
9 files changed, 356 insertions(+), 284 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 980e26082064f..e2f192a2c5bf8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -290,8 +290,6 @@ GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {
SIModeRegisterDefaults Mode(F, *ST);
HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
- HasFP64FP16Denormals =
- Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();
}
bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
@@ -524,6 +522,24 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
}
}
+bool GCNTTIImpl::canFuseFMulWithFAddSub(MVT::SimpleValueType SLT,
+ const Instruction *FMul,
+ const Instruction *FAddSub) const {
+ const int OPC = TLI->InstructionOpcodeToISD(FAddSub->getOpcode());
+ if (OPC != ISD::FADD && OPC != ISD::FSUB)
+ return false;
+
+ // The mad forms fuse exactly without fast-math flags but flush denormals.
+ const DenormalFPEnv FPEnv = FAddSub->getFunction()->getDenormalFPEnv();
+ if (TLI->isFMADLegal(MVT(SLT), FPEnv))
+ return true;
+
+ // Other types fuse only with contract or fast.
+ const TargetOptions &Options = TLI->getTargetMachine().Options;
+ return Options.AllowFPOpFusion == FPOpFusion::Fast ||
+ (FAddSub->hasAllowContract() && FMul->hasAllowContract());
+}
+
InstructionCost GCNTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
@@ -587,21 +603,9 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
// fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
// fused operation.
if (CxtI && CxtI->hasOneUse())
- if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin())) {
- const int OPC = TLI->InstructionOpcodeToISD(FAdd->getOpcode());
- if (OPC == ISD::FADD || OPC == ISD::FSUB) {
- if (ST->hasMadMacF32Insts() && SLT == MVT::f32 && !HasFP32Denormals)
- return TargetTransformInfo::TCC_Free;
- if (ST->has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)
- return TargetTransformInfo::TCC_Free;
-
- // Estimate all types may be fused with contract/unsafe flags
- const TargetOptions &Options = TLI->getTargetMachine().Options;
- if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
- (FAdd->hasAllowContract() && CxtI->hasAllowContract()))
- return TargetTransformInfo::TCC_Free;
- }
- }
+ if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin()))
+ if (canFuseFMulWithFAddSub(SLT, CxtI, FAdd))
+ return TargetTransformInfo::TCC_Free;
[[fallthrough]];
case ISD::FADD:
case ISD::FSUB:
@@ -1483,6 +1487,26 @@ bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const {
using namespace PatternMatch;
+ // The cost model prices this fmul as free assuming it fuses with its
+ // fadd/fsub user, which needs them in one block. Sink a stranded
+ // loop-invariant fmul back to the user when they would fuse. Single use only,
+ // so this stays a move.
+ if (I->getOpcode() == Instruction::FAdd ||
+ I->getOpcode() == Instruction::FSub) {
+ MVT::SimpleValueType SLT =
+ getTypeLegalizationCost(I->getType()).second.getScalarType().SimpleTy;
+ for (Use &Op : I->operands()) {
+ auto *FMul = dyn_cast<Instruction>(Op.get());
+ if (!FMul || FMul->getOpcode() != Instruction::FMul ||
+ !FMul->hasOneUse() || !canFuseFMulWithFAddSub(SLT, FMul, I))
+ continue;
+ // The fused operand. Sink it when it sits in another block, then stop.
+ if (FMul->getParent() != I->getParent())
+ Ops.push_back(&Op);
+ break;
+ }
+ }
+
for (auto &Op : I->operands()) {
// Ensure we are not already sinking this operand.
if (any_of(Ops, [&](Use *U) { return U->get() == Op.get(); }))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index df7b6d339e6c2..155dcca92f4ac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -71,7 +71,6 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
AMDGPUTTIImpl CommonTTI;
bool IsGraphics;
bool HasFP32Denormals;
- bool HasFP64FP16Denormals;
static constexpr bool InlinerVectorBonusPercent = 0;
static const FeatureBitset InlineFeatureIgnoreList;
@@ -103,6 +102,12 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
+ /// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
+ /// expected to fuse during instruction selection. \p SLT is the legalized
+ /// scalar type.
+ bool canFuseFMulWithFAddSub(MVT::SimpleValueType SLT, const Instruction *FMul,
+ const Instruction *FAddSub) const;
+
/// \returns true if V might be divergent even when all of its operands
/// are uniform.
bool isSourceOfDivergence(const Value *V) const;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
index 98773ba7292f4..858292b469fba 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fused_costs.ll
@@ -92,65 +92,65 @@ define void @fmul_fadd_f32() #0 {
}
define void @fmul_fadd_f16() #0 {
-; FUSED-LABEL: 'fmul_fadd_f16'
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FUSED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; SLOWF32-LABEL: 'fmul_fadd_f16'
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX9SLOW-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; GFX9SLOW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; FASTF32-LABEL: 'fmul_fadd_f16'
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; FASTF32-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; FUSED-SIZE-LABEL: 'fmul_fadd_f16'
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; FUSED-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+; SLOWF32-SIZE-LABEL: 'fmul_fadd_f16'
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16 = fmul half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16_2 = fmul half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; SLOWF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
-; GFX9SLOW-SIZE-LABEL: 'fmul_fadd_f16'
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
-; GFX9SLOW-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+; FASTF32-SIZE-LABEL: 'fmul_fadd_f16'
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16 = fmul half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16add = fadd half %f16, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c = fmul contract half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15cadd = fadd contract half %f16c, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fmul <2 x half> undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16add = fadd <2 x half> %v2f16, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16_2 = fmul half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f16sub = fsub half %f16_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %f16c_2 = fmul contract half undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %f15csub = fsub contract half %f16c_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16_2 = fmul <2 x half> undef, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16sub = fsub <2 x half> %v2f16_2, undef
+; FASTF32-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
%f16 = fmul half undef, undef
%f16add = fadd half %f16, undef
@@ -255,3 +255,8 @@ define void @fmul_fadd_f64() #0 {
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FUSED: {{.*}}
+; FUSED-SIZE: {{.*}}
+; GFX9SLOW: {{.*}}
+; GFX9SLOW-SIZE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
index a1da18969552a..d4870b7ca8a95 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
@@ -14,18 +14,17 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrsp
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: s_mov_b64 s[6:7], 0
-; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
+; GCN-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_load_dword s8, s[4:5], 0x0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s8
; GCN-NEXT: .LBB0_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_add_f32_e32 v4, v5, v2
+; GCN-NEXT: v_mad_f32 v4, 4.0, v2, v5
; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index d307ffaff5c63..8672b6eab90de 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -27,18 +27,17 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
@@ -63,17 +62,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX9-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -96,16 +94,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1064-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -128,15 +125,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1032-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
@@ -198,18 +194,17 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
@@ -234,17 +229,16 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX9-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -267,16 +261,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1064-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -299,15 +292,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX1032-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 327928e152f95..47891774dfabb 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -27,18 +27,17 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
@@ -63,17 +62,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX9-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -96,16 +94,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1064-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -128,15 +125,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1032-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
@@ -228,18 +224,17 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
@@ -264,17 +259,16 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX9-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -297,16 +291,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1064-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -329,15 +322,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX1032-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
index c01e3b436a3f1..c49906c0818dd 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -25,10 +25,11 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB0_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -44,7 +45,7 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -53,7 +54,7 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f32 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -63,14 +64,14 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_f32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB0_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_fmac_f32_e32 v0, v3, v1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -116,10 +117,11 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8-LABEL: fsub_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
; GFX8-NEXT: .LBB1_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -135,7 +137,7 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-LABEL: fsub_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -144,7 +146,7 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f32 v0, -v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -154,14 +156,14 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX10-LABEL: fsub_in_loop_f32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB1_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_fma_f32 v0, -v3, v1, v0
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -207,7 +209,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -216,7 +218,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_fma_f16 v0, v3, v1, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB2_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -226,7 +228,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -235,7 +237,7 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: v_fma_f16 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB2_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -245,14 +247,14 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_f16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB2_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -300,14 +302,13 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f64:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_mov_b32_e32 v6, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB3_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX8-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -315,19 +316,20 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
; GFX8-NEXT: ; %bb.2: ; %exit
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fma_in_loop_f64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB3_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -335,19 +337,20 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX9-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-NEXT: ; %bb.2: ; %exit
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v0, v5
+; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fma_in_loop_f64:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_mov_b32_e32 v6, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB3_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX10-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
; GFX10-NEXT: s_add_i32 s5, s5, 1
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -355,6 +358,8 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
; GFX10-NEXT: ; %bb.2: ; %exit
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
%mul = fmul contract double %a, %b
@@ -395,7 +400,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX8-LABEL: mad_in_loop_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -404,7 +409,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_mac_f32_e32 v0, v3, v1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -414,7 +419,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX9-LABEL: mad_in_loop_f32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -423,7 +428,7 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mac_f32_e32 v0, v3, v1
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -658,7 +663,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX8-LABEL: mad_in_loop_f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
@@ -667,7 +672,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX8-NEXT: s_add_i32 s6, s6, 1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -677,7 +682,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX9-LABEL: mad_in_loop_f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -686,7 +691,7 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -732,20 +737,20 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX8-LABEL: no_fma_in_loop_v2f16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB8_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_mac_f16_sdwa v4, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT: v_mac_f16_e32 v0, v3, v1
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %exit
@@ -755,10 +760,11 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX9-LABEL: no_fma_in_loop_v2f16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX9-NEXT: .LBB8_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -774,10 +780,11 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX10-LABEL: no_fma_in_loop_v2f16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX10-NEXT: .LBB8_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
@@ -809,30 +816,32 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX8-LABEL: fma_in_loop_v2f16_denormals:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB9_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_fma_f16 v4, v6, v5, v4
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-NEXT: v_fma_f16 v3, v0, v1, v3
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %exit
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fma_in_loop_v2f16_denormals:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -841,7 +850,7 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX9-NEXT: s_add_i32 s6, s6, 1
; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT: v_pk_fma_f16 v0, v3, v1, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_1
; GFX9-NEXT: ; %bb.2: ; %exit
@@ -851,14 +860,14 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX10-LABEL: fma_in_loop_v2f16_denormals:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB9_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX10-NEXT: v_pk_fma_f16 v0, v3, v1, v0
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
@@ -887,31 +896,23 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_bf16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: .LBB10_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX8-NEXT: v_fma_f32 v0, v3, v1, v0
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX8-NEXT: s_add_i32 s6, s6, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -924,31 +925,24 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_bf16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s6, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s6
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_mov_b32 s7, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: s_movk_i32 s7, 0x7fff
; GFX9-NEXT: .LBB10_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX9-NEXT: v_add3_u32 v3, v3, v0, s6
-; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT: v_fma_f32 v0, v3, v1, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s7
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: s_add_i32 s7, s7, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s7, v2
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -960,30 +954,23 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX10-LABEL: fma_in_loop_bf16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: .p2align 6
; GFX10-NEXT: .LBB10_1: ; %loop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_add_i32 s5, s5, 1
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_fmac_f32_e32 v0, v3, v1
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
index a8a4d12a502e3..2fd2600197aa1 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
@@ -7,9 +7,9 @@ define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fadd_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
index b548b99663f9e..602d9ebd0dadc 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
@@ -8,9 +8,9 @@ define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fadd_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -34,9 +34,9 @@ define float @sink_fmul_fsub_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fsub_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -61,9 +61,9 @@ define float @sink_fmul_fsub_rev_f32(i1 %cond, float %a, float %b, float %c) {
; CHECK-LABEL: define float @sink_fmul_fsub_rev_f32(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[MUL]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -87,9 +87,9 @@ define half @sink_fmul_fadd_f16(i1 %cond, half %a, half %b, half %c) {
; CHECK-LABEL: define half @sink_fmul_fadd_f16(
; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract half [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract half [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract half [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -113,9 +113,9 @@ define double @sink_fmul_fadd_f64(i1 %cond, double %a, double %b, double %c) {
; CHECK-LABEL: define double @sink_fmul_fadd_f64(
; CHECK-SAME: i1 [[COND:%.*]], double [[A:%.*]], double [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract double [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract double [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract double [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -139,9 +139,9 @@ define <2 x float> @sink_fmul_fadd_v2f32(i1 %cond, <2 x float> %a, <2 x float> %
; CHECK-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
; CHECK-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -166,10 +166,10 @@ define float @sink_only_fusable_fmul(i1 %cond, float %a, float %b, float %c, flo
; CHECK-LABEL: define float @sink_only_fusable_fmul(
; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -221,17 +221,41 @@ exit:
; v_mad_f32 is exact, so with denormals flushed no fast-math flags are needed.
define float @sink_fmul_fadd_f32_no_denormals(i1 %cond, float %a, float %b, float %c) #0 {
-; CHECK-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret float [[R]]
+; GFX8-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX8-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret float [[R]]
+;
+; GFX9-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX9-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret float [[R]]
+;
+; GFX10-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; GFX10-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret float [[R]]
;
entry:
%mul = fmul float %a, %b
@@ -248,17 +272,41 @@ exit:
; Same for v_mad_f16.
define half @sink_fmul_fadd_f16_no_denormals(i1 %cond, half %a, half %b, half %c) #0 {
-; CHECK-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul half [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd half [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: ret half [[R]]
+; GFX8-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX8-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul half [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd half [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX8-NEXT: ret half [[R]]
+;
+; GFX9-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX9-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul half [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd half [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX9-NEXT: ret half [[R]]
+;
+; GFX10-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; GFX10-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul half [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd half [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; GFX10-NEXT: ret half [[R]]
;
entry:
%mul = fmul half %a, %b
@@ -428,11 +476,11 @@ define float @sink_fmul_into_loop(float %a, float %b, i32 %n) {
; CHECK-LABEL: define float @sink_fmul_into_loop(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT: [[ADD]] = fadd contract float [[ACC]], [[MUL]]
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
@@ -459,11 +507,11 @@ exit:
; A legal v2f16 has no packed mad, and v_pk_fma_f16 needs denormals enabled.
define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[MUL]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -485,17 +533,41 @@ exit:
; Same without the contract flags.
define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
-; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
-; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x half> [[A]], [[B]]
-; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: [[ADD:%.*]] = fadd <2 x half> [[MUL]], [[C]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT: ret <2 x half> [[R]]
+; GFX8-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX8-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX8: [[IF]]:
+; GFX8-NEXT: [[TMP0:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX8-NEXT: [[ADD:%.*]] = fadd <2 x half> [[TMP0]], [[C]]
+; GFX8-NEXT: br label %[[EXIT]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX8-NEXT: ret <2 x half> [[R]]
+;
+; GFX9-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX9-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX9: [[IF]]:
+; GFX9-NEXT: [[TMP0:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX9-NEXT: [[ADD:%.*]] = fadd <2 x half> [[TMP0]], [[C]]
+; GFX9-NEXT: br label %[[EXIT]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX9-NEXT: ret <2 x half> [[R]]
+;
+; GFX10-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; GFX10-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT: [[ENTRY:.*]]:
+; GFX10-NEXT: [[MUL:%.*]] = fmul <2 x half> [[A]], [[B]]
+; GFX10-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; GFX10: [[IF]]:
+; GFX10-NEXT: [[ADD:%.*]] = fadd <2 x half> [[MUL]], [[C]]
+; GFX10-NEXT: br label %[[EXIT]]
+; GFX10: [[EXIT]]:
+; GFX10-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; GFX10-NEXT: ret <2 x half> [[R]]
;
entry:
%mul = fmul <2 x half> %a, %b
@@ -515,9 +587,9 @@ define <2 x half> @sink_fmul_fadd_v2f16_denormals(i1 %cond, <2 x half> %a, <2 x
; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_denormals(
; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -542,9 +614,9 @@ define bfloat @sink_fmul_fadd_bf16(i1 %cond, bfloat %a, bfloat %b, bfloat %c) {
; CHECK-LABEL: define bfloat @sink_fmul_fadd_bf16(
; CHECK-SAME: i1 [[COND:%.*]], bfloat [[A:%.*]], bfloat [[B:%.*]], bfloat [[C:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract bfloat [[A]], [[B]]
; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
; CHECK: [[IF]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract bfloat [[A]], [[B]]
; CHECK-NEXT: [[ADD:%.*]] = fadd contract bfloat [[TMP0]], [[C]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -565,7 +637,3 @@ exit:
}
attributes #0 = { denormal_fpenv(preservesign) }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX8: {{.*}}
-; GFX9: {{.*}}
More information about the llvm-commits
mailing list