[llvm-branch-commits] [llvm] [AMDGPU][DAGCombiner][GlobalISel] Extend allMulUsesCanBeContracted with FMA/FMAD pattern (PR #188117)
Adel Ejjeh via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Mar 30 08:54:39 PDT 2026
https://github.com/adelejjeh updated https://github.com/llvm/llvm-project/pull/188117
>From 4ba5032d5357180f1a69970b247d632792e8eb17 Mon Sep 17 00:00:00 2001
From: Adel Ejjeh <adel.ejjeh at amd.com>
Date: Thu, 12 Mar 2026 11:11:20 -0500
Subject: [PATCH] [AMDGPU][DAGCombiner][GlobalISel] Extend
allMulUsesCanBeContracted with FMA/FMAD pattern
Add conservative FMA/FMAD recognition to allMulUsesCanBeContracted:
a multiply used by an existing FMA/FMAD is assumed to be contractable
(it's already being contracted elsewhere). This avoids unnecessary
contraction blocking for multiplies that feed into FMA chains.
Also adds FMA/FMAD to the FPEXT user set (fpext(fmul) --> fma is
recognized as contractable when isFPExtFoldable).
Guards all remaining FMA-chain reassociation fold sites in both
SDAG (visitFADDForFMACombine/visitFSUBForFMACombine, 8 sites) and
GISel (matchCombineFAddFpExtFMulToFMadOrFMAAggressive, 4 sites).
This re-enables contractions that were conservatively blocked in
earlier patches where the multiply had an FMA use that wasn't yet
recognized: dagcombine-fma-crash.ll and dagcombine-fma-fmad.ll
CHECK lines revert to upstream behavior.
Co-Authored-By: Claude Opus 4.6 <noreply at anthropic.com>
Made-with: Cursor
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 58 ++-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 56 ++-
.../CodeGen/AMDGPU/dagcombine-fma-crash.ll | 22 +-
.../CodeGen/AMDGPU/dagcombine-fma-fmad.ll | 169 ++++-----
.../AMDGPU/fma-multiple-uses-contraction.ll | 359 +++++++++---------
5 files changed, 374 insertions(+), 290 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 0941e6da0f40f..dab61caeeacfb 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6321,7 +6321,10 @@ static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
/// - fmul --> fneg --> fsub: Contraction through fneg
/// - fmul --> fneg --> fpext --> fsub: FNEG then FPEXT folds if foldable
/// - fmul --> fpext --> {fadd, fsub}: FPEXT folds if foldable
+/// - fmul --> fpext --> {fma, fmad} --> {fadd, fsub}: FPEXT chain reassoc
/// - fmul --> fpext --> fneg --> fsub: FPEXT then FNEG to FSUB
+/// - fmul --> {fma, fmad} --> {fadd, fsub} (reassoc): Direct chain reassoc
+/// - fmul --> {fma, fmad} --> fpext --> {fadd, fsub}: FPEXT outer chain
bool CombinerHelper::allMulUsesCanBeContracted(
const MachineInstr &MI, unsigned PreferredFusedOpcode) const {
const auto &TLI = getTargetLowering();
@@ -6367,7 +6370,8 @@ bool CombinerHelper::allMulUsesCanBeContracted(
continue;
}
- // FP_EXTEND - check if ALL users are FADD, FSUB, or FNEG --> FSUB
+ // FP_EXTEND - check if ALL users are FADD, FSUB, FNEG --> FSUB, or
+ // FMA/FMAD --> {FADD, FSUB}
if (Opcode == TargetOpcode::G_FPEXT) {
Register FPExtReg = UseMI.getOperand(0).getReg();
@@ -6383,6 +6387,26 @@ bool CombinerHelper::allMulUsesCanBeContracted(
ExtUseOpcode == TargetOpcode::G_FSUB) {
continue;
}
+ if (ExtUseOpcode == TargetOpcode::G_FMA ||
+ ExtUseOpcode == TargetOpcode::G_FMAD) {
+ // FPEXT --> FMA/FMAD is only contractable if the FMA/FMAD is
+ // used by FADD or FSUB (chain reassociation can fire to
+ // eliminate the multiply).
+ Register FMAReg = FPExtUseMI.getOperand(0).getReg();
+ bool FMAUsedByAddSub = false;
+ for (const MachineInstr &FMAUseMI :
+ MRI.use_nodbg_instructions(FMAReg)) {
+ unsigned FMAUseOp = FMAUseMI.getOpcode();
+ if (FMAUseOp == TargetOpcode::G_FADD ||
+ FMAUseOp == TargetOpcode::G_FSUB) {
+ FMAUsedByAddSub = true;
+ break;
+ }
+ }
+ if (FMAUsedByAddSub)
+ continue;
+ return false;
+ }
if (ExtUseOpcode == TargetOpcode::G_FNEG) {
// FP_EXTEND --> FNEG --> FSUB
Register FPExtFNegReg = FPExtUseMI.getOperand(0).getReg();
@@ -6398,6 +6422,34 @@ bool CombinerHelper::allMulUsesCanBeContracted(
continue;
}
+ // FMA/FMAD - the multiply is used as an operand of an FMA. This is
+ // contractable only if chain reassociation can fire to eliminate the
+ // multiply. Chain reassociation transforms:
+ // fadd/fsub(fma(a, b, fmul(c, d)), e) -> fma(a, b, fma(c, d, e))
+ // For direct fmul -> fma -> fadd/fsub, this requires the consumer
+ // fadd/fsub to have the reassoc flag. For fmul -> fma -> fpext ->
+ // fadd/fsub, the aggressive fpext folds handle it without reassoc.
+ if (Opcode == TargetOpcode::G_FMA || Opcode == TargetOpcode::G_FMAD) {
+ Register FMAReg = UseMI.getOperand(0).getReg();
+ bool FMAIsContractable = false;
+ for (const MachineInstr &FMAUseMI : MRI.use_nodbg_instructions(FMAReg)) {
+ unsigned FMAUseOp = FMAUseMI.getOpcode();
+ if (FMAUseOp == TargetOpcode::G_FPEXT) {
+ FMAIsContractable = true;
+ break;
+ }
+ if ((FMAUseOp == TargetOpcode::G_FADD ||
+ FMAUseOp == TargetOpcode::G_FSUB) &&
+ FMAUseMI.getFlag(MachineInstr::MIFlag::FmReassoc)) {
+ FMAIsContractable = true;
+ break;
+ }
+ }
+ if (FMAIsContractable)
+ continue;
+ return false;
+ }
+
// Any other use type is not currently recognized as contractable.
return false;
}
@@ -6670,6 +6722,7 @@ bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
mi_match(LHS.MI->getOperand(3).getReg(), MRI,
m_GFPExt(m_MInstr(FMulMI))) &&
isContractableFMul(*FMulMI, AllowFusionGlobally) &&
+ allMulUsesCanBeContracted(*FMulMI, PreferredFusedOpcode) &&
TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
MRI.getType(FMulMI->getOperand(0).getReg()))) {
MatchInfo = [=](MachineIRBuilder &B) {
@@ -6690,6 +6743,7 @@ bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
FMAMI->getOpcode() == PreferredFusedOpcode) {
MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg());
if (isContractableFMul(*FMulMI, AllowFusionGlobally) &&
+ allMulUsesCanBeContracted(*FMulMI, PreferredFusedOpcode) &&
TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
MRI.getType(FMAMI->getOperand(0).getReg()))) {
MatchInfo = [=](MachineIRBuilder &B) {
@@ -6711,6 +6765,7 @@ bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
mi_match(RHS.MI->getOperand(3).getReg(), MRI,
m_GFPExt(m_MInstr(FMulMI))) &&
isContractableFMul(*FMulMI, AllowFusionGlobally) &&
+ allMulUsesCanBeContracted(*FMulMI, PreferredFusedOpcode) &&
TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
MRI.getType(FMulMI->getOperand(0).getReg()))) {
MatchInfo = [=](MachineIRBuilder &B) {
@@ -6731,6 +6786,7 @@ bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
FMAMI->getOpcode() == PreferredFusedOpcode) {
MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg());
if (isContractableFMul(*FMulMI, AllowFusionGlobally) &&
+ allMulUsesCanBeContracted(*FMulMI, PreferredFusedOpcode) &&
TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
MRI.getType(FMAMI->getOperand(0).getReg()))) {
MatchInfo = [=](MachineIRBuilder &B) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e4f1c8adc8abe..b307a724ba2cb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17690,7 +17690,10 @@ static bool isFusedOp(const MatchContextClass &Matcher, SDValue N) {
/// - fmul --> fneg --> fsub: Contraction through fneg
/// - fmul --> fneg --> fpext --> fsub: FNEG then FPEXT folds if foldable
/// - fmul --> fpext --> {fadd, fsub}: FPEXT folds if foldable
+/// - fmul --> fpext --> {fma, fmad} --> {fadd, fsub}: FPEXT chain reassoc
/// - fmul --> fpext --> fneg --> fsub: FPEXT then FNEG to FSUB
+/// - fmul --> {fma, fmad} --> {fadd, fsub} (reassoc): Direct chain reassoc
+/// - fmul --> {fma, fmad} --> fpext --> {fadd, fsub}: FPEXT outer chain
static bool allMulUsesCanBeContracted(SDValue Mul,
const unsigned PreferredFusedOpcode,
const TargetLowering &TLI,
@@ -17729,7 +17732,8 @@ static bool allMulUsesCanBeContracted(SDValue Mul,
continue; // All FNEG uses are contractable
}
- // FP_EXTEND - check if ALL users are FADD, FSUB, or FNEG --> FSUB
+ // FP_EXTEND - check if ALL users are FADD, FSUB, FNEG --> FSUB, or
+ // FMA/FMAD --> {FADD, FSUB}
if (Opcode == ISD::FP_EXTEND) {
EVT SrcVT = Mul.getValueType();
@@ -17742,6 +17746,22 @@ static bool allMulUsesCanBeContracted(SDValue Mul,
if (ExtUserOp == ISD::FADD || ExtUserOp == ISD::FSUB) {
continue; // FPEXT --> {FADD, FSUB} is contractable
}
+ if (ExtUserOp == ISD::FMA || ExtUserOp == ISD::FMAD) {
+ // FPEXT --> FMA/FMAD is only contractable if the FMA/FMAD is
+ // used by FADD or FSUB (chain reassociation can fire to
+ // eliminate the multiply).
+ bool FMAUsedByAddSub = false;
+ for (const auto *FMAUser : FPExtUser->users()) {
+ unsigned FMAUserOp = FMAUser->getOpcode();
+ if (FMAUserOp == ISD::FADD || FMAUserOp == ISD::FSUB) {
+ FMAUsedByAddSub = true;
+ break;
+ }
+ }
+ if (FMAUsedByAddSub)
+ continue;
+ return false;
+ }
if (ExtUserOp == ISD::FNEG) {
// FP_EXTEND --> FNEG --> FSUB
for (const auto *FPExtFNegUser : FPExtUser->users()) {
@@ -17756,6 +17776,32 @@ static bool allMulUsesCanBeContracted(SDValue Mul,
continue; // All FPEXT uses are contractable
}
+ // FMA/FMAD - the multiply is used as an operand of an FMA. This is
+ // contractable only if chain reassociation can fire to eliminate the
+ // multiply. Chain reassociation transforms:
+ // fadd/fsub(fma(a, b, fmul(c, d)), e) -> fma(a, b, fma(c, d, e))
+ // For direct fmul -> fma -> fadd/fsub, this requires the consumer
+ // fadd/fsub to have the reassoc flag. For fmul -> fma -> fpext ->
+ // fadd/fsub, the aggressive fpext folds handle it without reassoc.
+ if (Opcode == ISD::FMA || Opcode == ISD::FMAD) {
+ bool FMAIsContractable = false;
+ for (const auto *FMAUser : UserNode->users()) {
+ unsigned FMAUserOp = FMAUser->getOpcode();
+ if (FMAUserOp == ISD::FP_EXTEND) {
+ FMAIsContractable = true;
+ break;
+ }
+ if ((FMAUserOp == ISD::FADD || FMAUserOp == ISD::FSUB) &&
+ FMAUser->getFlags().hasAllowReassociation()) {
+ FMAIsContractable = true;
+ break;
+ }
+ }
+ if (FMAIsContractable)
+ continue;
+ return false;
+ }
+
// Any other use type is not currently recognized as contractable.
return false;
}
@@ -17930,6 +17976,7 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
if (matcher.match(N02, ISD::FP_EXTEND)) {
SDValue N020 = N02.getOperand(0);
if (isContractableFMUL(N020) &&
+ allMulUsesCanBeContracted(N020, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N020.getValueType())) {
return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1),
@@ -17959,6 +18006,7 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
if (isFusedOp(matcher, N00)) {
SDValue N002 = N00.getOperand(2);
if (isContractableFMUL(N002) &&
+ allMulUsesCanBeContracted(N002, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N00.getValueType())) {
return FoldFAddFPExtFMAFMul(N00.getOperand(0), N00.getOperand(1),
@@ -17975,6 +18023,7 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
if (N12.getOpcode() == ISD::FP_EXTEND) {
SDValue N120 = N12.getOperand(0);
if (isContractableFMUL(N120) &&
+ allMulUsesCanBeContracted(N120, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N120.getValueType())) {
return FoldFAddFMAFPExtFMul(N1.getOperand(0), N1.getOperand(1),
@@ -17994,6 +18043,7 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
if (isFusedOp(matcher, N10)) {
SDValue N102 = N10.getOperand(2);
if (isContractableFMUL(N102) &&
+ allMulUsesCanBeContracted(N102, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N10.getValueType())) {
return FoldFAddFPExtFMAFMul(N10.getOperand(0), N10.getOperand(1),
@@ -18256,6 +18306,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
if (matcher.match(N02, ISD::FP_EXTEND)) {
SDValue N020 = N02.getOperand(0);
if (isContractableAndReassociableFMUL(N020) &&
+ allMulUsesCanBeContracted(N020, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N020.getValueType())) {
return matcher.getNode(
@@ -18280,6 +18331,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
if (isFusedOp(matcher, N00)) {
SDValue N002 = N00.getOperand(2);
if (isContractableAndReassociableFMUL(N002) &&
+ allMulUsesCanBeContracted(N002, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N00.getValueType())) {
return matcher.getNode(
@@ -18301,6 +18353,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
matcher.match(N1.getOperand(2), ISD::FP_EXTEND) && N1->hasOneUse()) {
SDValue N120 = N1.getOperand(2).getOperand(0);
if (isContractableAndReassociableFMUL(N120) &&
+ allMulUsesCanBeContracted(N120, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
N120.getValueType())) {
SDValue N1200 = N120.getOperand(0);
@@ -18330,6 +18383,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDValue N101 = CvtSrc.getOperand(1);
SDValue N102 = CvtSrc.getOperand(2);
if (isContractableAndReassociableFMUL(N102) &&
+ allMulUsesCanBeContracted(N102, PreferredFusedOpcode, TLI, DAG) &&
TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
CvtSrc.getValueType())) {
SDValue N1020 = N102.getOperand(0);
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
index 57070e763e79b..142494a803755 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
@@ -20,24 +20,22 @@ define void @main(float %arg) {
; CHECK-NEXT: bb.1.bb2:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MUL_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_MUL_F32_e64 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 1065353216
- ; CHECK-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_ADD_F32_e64 0, [[V_MUL_F32_e64_]], 0, [[S_MOV_B32_2]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_FMAC_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY]], 0, [[V_ADD_F32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_FMAC_F32_e64_1:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY]], 0, [[V_MUL_F32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_ADD_F32_e64 0, killed [[V_FMAC_F32_e64_1]], 0, [[S_MOV_B32_2]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ ; CHECK-NEXT: [[V_FMAC_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_FMAC_F32_e64_1:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY]], 0, [[V_FMAC_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = contract reassoc nofpexcept V_ADD_F32_e64 0, [[V_FMAC_F32_e64_1]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.bb11:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.4(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_FMAC_F32_e64_]], %bb.1
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_ADD_F32_e64_1]], %bb.1
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, [[S_MOV_B32_3]], %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_FMAC_F32_e64_1]], %bb.1
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, [[V_ADD_F32_e64_]], %bb.1
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, [[S_MOV_B32_2]], %bb.1
; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[PHI2]], implicit $exec
- ; CHECK-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+ ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[V_CNDMASK_B32_e64_]]
- ; CHECK-NEXT: S_CMP_LG_U32 killed [[COPY1]], killed [[S_MOV_B32_4]], implicit-def $scc
+ ; CHECK-NEXT: S_CMP_LG_U32 killed [[COPY1]], killed [[S_MOV_B32_3]], implicit-def $scc
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $scc
; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 $exec_lo, killed [[COPY2]], implicit-def dead $scc
; CHECK-NEXT: $vcc_lo = COPY [[S_AND_B32_1]]
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index e95f19a19d134..8e0e7d2a735f8 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -17,57 +17,54 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
-; GFX10-NEXT: s_clause 0x2
+; GFX10-NEXT: s_clause 0x3
; GFX10-NEXT: s_buffer_load_dword s24, s[0:3], 0x5c
; GFX10-NEXT: s_buffer_load_dword s25, s[0:3], 0x7c
+; GFX10-NEXT: s_buffer_load_dword s28, s[0:3], 0xc0
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_nop 0
-; GFX10-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_buffer_load_dword s26, s[0:3], 0xc0
-; GFX10-NEXT: s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50
-; GFX10-NEXT: s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60
+; GFX10-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x70
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_clause 0x3
-; GFX10-NEXT: s_buffer_load_dword s4, s[0:3], 0x2c
-; GFX10-NEXT: s_buffer_load_dwordx4 s[12:15], s[0:3], 0x70
-; GFX10-NEXT: s_buffer_load_dwordx4 s[16:19], s[0:3], 0x20
-; GFX10-NEXT: s_buffer_load_dwordx4 s[20:23], s[0:3], 0x0
-; GFX10-NEXT: v_max_f32_e64 v5, s0, s0 clamp
-; GFX10-NEXT: v_sub_f32_e64 v6, s24, s25
-; GFX10-NEXT: v_mul_f32_e32 v7, s2, v5
-; GFX10-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x10
-; GFX10-NEXT: v_fma_f32 v1, v1, v6, s25
+; GFX10-NEXT: v_sub_f32_e64 v5, s24, s25
+; GFX10-NEXT: s_buffer_load_dword s0, s[0:3], 0x2c
+; GFX10-NEXT: v_mul_f32_e32 v2, s2, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_clause 0x4
+; GFX10-NEXT: s_buffer_load_dwordx4 s[4:7], s[0:3], 0x40
+; GFX10-NEXT: s_buffer_load_dwordx4 s[8:11], s[0:3], 0x50
+; GFX10-NEXT: s_buffer_load_dwordx4 s[12:15], s[0:3], 0x0
+; GFX10-NEXT: s_buffer_load_dwordx4 s[16:19], s[0:3], 0x60
+; GFX10-NEXT: s_buffer_load_dwordx4 s[20:23], s[0:3], 0x20
+; GFX10-NEXT: v_fma_f32 v1, v1, v5, s25
+; GFX10-NEXT: s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10
+; GFX10-NEXT: v_max_f32_e64 v6, s0, s0 clamp
+; GFX10-NEXT: v_add_f32_e64 v5, s28, -1.0
+; GFX10-NEXT: v_sub_f32_e32 v7, s0, v1
; GFX10-NEXT: s_mov_b32 s0, 0x3c23d70a
-; GFX10-NEXT: v_add_f32_e64 v6, s26, -1.0
-; GFX10-NEXT: v_sub_f32_e32 v8, s6, v7
-; GFX10-NEXT: v_mul_f32_e32 v0, s10, v0
-; GFX10-NEXT: v_sub_f32_e32 v9, s4, v1
-; GFX10-NEXT: v_mul_f32_e32 v2, s14, v2
-; GFX10-NEXT: v_fma_f32 v6, v5, v6, 1.0
-; GFX10-NEXT: v_fmac_f32_e32 v7, v8, v5
-; GFX10-NEXT: v_sub_f32_e32 v8, s18, v0
-; GFX10-NEXT: v_fmac_f32_e32 v1, v5, v9
-; GFX10-NEXT: v_mul_f32_e32 v9, s22, v3
-; GFX10-NEXT: v_mul_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_add_f32_e32 v7, v3, v7
-; GFX10-NEXT: v_fmac_f32_e32 v0, v8, v5
-; GFX10-NEXT: v_sub_f32_e32 v1, v1, v6
-; GFX10-NEXT: v_mul_f32_e32 v8, v9, v5
-; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_fmac_f32_e32 v6, v1, v5
-; GFX10-NEXT: v_fma_f32 v1, v3, s2, -v8
-; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v5
-; GFX10-NEXT: v_fmaak_f32 v0, s0, v6, 0x3ca3d70a
-; GFX10-NEXT: v_fmac_f32_e32 v8, v1, v5
-; GFX10-NEXT: v_mul_f32_e32 v1, v3, v5
+; GFX10-NEXT: v_mul_f32_e32 v2, v6, v2
+; GFX10-NEXT: v_fma_f32 v5, v6, v5, 1.0
+; GFX10-NEXT: v_fmac_f32_e32 v1, v6, v7
+; GFX10-NEXT: v_sub_f32_e32 v1, v1, v5
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_fma_f32 v8, -s6, v6, s10
+; GFX10-NEXT: v_mad_f32 v9, s6, v6, v3
+; GFX10-NEXT: v_mul_f32_e32 v7, s14, v3
+; GFX10-NEXT: v_fma_f32 v10, -v0, s18, s22
+; GFX10-NEXT: v_mad_f32 v0, v0, s18, -v2
+; GFX10-NEXT: v_fmac_f32_e32 v5, v1, v6
+; GFX10-NEXT: v_fmac_f32_e32 v9, v8, v6
+; GFX10-NEXT: v_mul_f32_e32 v7, v7, v6
+; GFX10-NEXT: v_mac_f32_e32 v0, v10, v6
+; GFX10-NEXT: v_fma_f32 v1, v3, s26, -v7
+; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v6
+; GFX10-NEXT: v_fmaak_f32 v0, s0, v5, 0x3ca3d70a
+; GFX10-NEXT: v_fmac_f32_e32 v7, v1, v6
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v6
; GFX10-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX10-NEXT: v_mul_f32_e32 v4, v4, v5
-; GFX10-NEXT: v_mul_f32_e32 v2, v4, v8
+; GFX10-NEXT: v_add_f32_e32 v4, v4, v9
+; GFX10-NEXT: v_mul_f32_e32 v4, v4, v6
+; GFX10-NEXT: v_mul_f32_e32 v2, v4, v7
; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1
; GFX10-NEXT: v_max_f32_e32 v0, 0, v2
; GFX10-NEXT: ; return to shader part epilog
@@ -85,62 +82,56 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
-; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: s_buffer_load_b32 s24, s[0:3], 0x5c
; GFX11-NEXT: s_buffer_load_b32 s25, s[0:3], 0x7c
-; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x40
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_buffer_load_b32 s26, s[0:3], 0xc0
-; GFX11-NEXT: s_buffer_load_b128 s[4:7], s[0:3], 0x50
-; GFX11-NEXT: s_buffer_load_b128 s[8:11], s[0:3], 0x60
+; GFX11-NEXT: s_buffer_load_b32 s28, s[0:3], 0xc0
+; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x70
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_buffer_load_b32 s4, s[0:3], 0x2c
-; GFX11-NEXT: s_buffer_load_b128 s[12:15], s[0:3], 0x70
-; GFX11-NEXT: s_buffer_load_b128 s[16:19], s[0:3], 0x20
-; GFX11-NEXT: s_buffer_load_b128 s[20:23], s[0:3], 0x0
-; GFX11-NEXT: v_max_f32_e64 v5, s0, s0 clamp
-; GFX11-NEXT: v_sub_f32_e64 v6, s24, s25
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_f32_e32 v7, s2, v5
-; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x10
-; GFX11-NEXT: v_fma_f32 v1, v1, v6, s25
+; GFX11-NEXT: v_sub_f32_e64 v5, s24, s25
+; GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x2c
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x4
+; GFX11-NEXT: s_buffer_load_b128 s[4:7], s[0:3], 0x40
+; GFX11-NEXT: s_buffer_load_b128 s[8:11], s[0:3], 0x50
+; GFX11-NEXT: s_buffer_load_b128 s[12:15], s[0:3], 0x0
+; GFX11-NEXT: s_buffer_load_b128 s[16:19], s[0:3], 0x60
+; GFX11-NEXT: s_buffer_load_b128 s[20:23], s[0:3], 0x20
+; GFX11-NEXT: v_fma_f32 v1, v1, v5, s25
+; GFX11-NEXT: v_max_f32_e64 v6, s0, s0 clamp
+; GFX11-NEXT: v_add_f32_e64 v5, s28, -1.0
+; GFX11-NEXT: s_buffer_load_b128 s[24:27], s[0:3], 0x10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_sub_f32 v7, s0, v1 :: v_dual_mul_f32 v2, s2, v2
+; GFX11-NEXT: v_fma_f32 v5, v6, v5, 1.0
; GFX11-NEXT: s_mov_b32 s0, 0x3c23d70a
-; GFX11-NEXT: v_add_f32_e64 v6, s26, -1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_sub_f32 v8, s6, v7 :: v_dual_sub_f32 v9, s4, v1
-; GFX11-NEXT: v_mul_f32_e32 v0, s10, v0
-; GFX11-NEXT: v_mul_f32_e32 v2, s14, v2
-; GFX11-NEXT: v_fma_f32 v6, v5, v6, 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_fmac_f32_e32 v7, v8, v5
-; GFX11-NEXT: v_dual_fmac_f32 v1, v5, v9 :: v_dual_sub_f32 v8, s18, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v9, s22, v3 :: v_dual_mul_f32 v2, v5, v2
-; GFX11-NEXT: v_add_f32_e32 v7, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_sub_f32 v1, v1, v6 :: v_dual_fmac_f32 v0, v8, v5
-; GFX11-NEXT: v_mul_f32_e32 v8, v9, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_fmac_f32_e32 v6, v1, v5
-; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fma_f32 v1, v3, s2, -v8
-; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_fmaak_f32 v0, s0, v6, 0x3ca3d70a
-; GFX11-NEXT: v_fmac_f32_e32 v8, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mul_f32 v1, v3, v5 :: v_dual_mul_f32 v0, v2, v0
+; GFX11-NEXT: v_fmac_f32_e32 v1, v6, v7
+; GFX11-NEXT: v_mul_f32_e32 v2, v6, v2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_fma_f32 v9, s6, v6, v3
+; GFX11-NEXT: v_fma_f32 v8, -s6, v6, s10
+; GFX11-NEXT: v_mul_f32_e32 v7, s14, v3
+; GFX11-NEXT: v_fma_f32 v10, -v0, s18, s22
+; GFX11-NEXT: v_fma_f32 v0, v0, s18, -v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_sub_f32 v1, v1, v5 :: v_dual_fmac_f32 v0, v10, v6
+; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v5, v1, v6
+; GFX11-NEXT: v_dual_fmac_f32 v9, v8, v6 :: v_dual_fmaak_f32 v0, s0, v5, 0x3ca3d70a
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mul_f32 v7, v7, v6 :: v_dual_mul_f32 v0, v2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f32_e32 v4, v4, v7
-; GFX11-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX11-NEXT: v_add_f32_e32 v4, v4, v9
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fma_f32 v1, v3, s26, -v7
+; GFX11-NEXT: v_mul_f32_e32 v4, v4, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v7, v1, v6
+; GFX11-NEXT: v_dual_mul_f32 v1, v3, v6 :: v_dual_mul_f32 v2, v4, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f32_e32 v2, v4, v8
; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_max_f32_e32 v0, 0, v2
; GFX11-NEXT: ; return to shader part epilog
.entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fma-multiple-uses-contraction.ll b/llvm/test/CodeGen/AMDGPU/fma-multiple-uses-contraction.ll
index 4fa74d8c9669d..081351957d496 100644
--- a/llvm/test/CodeGen/AMDGPU/fma-multiple-uses-contraction.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma-multiple-uses-contraction.ll
@@ -2452,12 +2452,6 @@ entry:
; ==========================================================================
; FMA/FMAD chain patterns
; Tests for allMulUsesCanBeContracted recognizing FMA/FMAD as contractable.
-;
-; NOTE: The allMulUsesCanBeContracted guard does not yet recognize FMA/FMAD
-; users of the multiply. That support is added by the final patch in the
-; series. Until then, the CHECK lines below reflect current (potentially
-; over-conservative) codegen and may not match the "Expected:" comments on
-; individual tests.
; ==========================================================================
; Test case: fpext(fmul) -> {fma -> fadd, fadd} (chained fma with fpext).
@@ -2588,10 +2582,10 @@ define {float, float} @fma_chain_fpext_outer_contractable(half %x, half %y, half
; GFX9-SDAG-LABEL: fma_chain_fpext_outer_contractable:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-SDAG-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, v6
; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-SDAG-NEXT: v_add_f16_e32 v1, v2, v5
+; GFX9-SDAG-NEXT: v_fma_f16 v1, v2, v3, v5
; GFX9-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX9-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -2599,10 +2593,10 @@ define {float, float} @fma_chain_fpext_outer_contractable(half %x, half %y, half
; GFX9-GISEL-LABEL: fma_chain_fpext_outer_contractable:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v6
; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX9-GISEL-NEXT: v_add_f16_e32 v1, v2, v5
+; GFX9-GISEL-NEXT: v_fma_f16 v1, v2, v3, v5
; GFX9-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX9-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2654,10 +2648,10 @@ define {float, float} @fma_chain_fpext_outer_contractable(half %x, half %y, half
; GFX9_4-SDAG-F32DENORM-LABEL: fma_chain_fpext_outer_contractable:
; GFX9_4-SDAG-F32DENORM: ; %bb.0: ; %entry
; GFX9_4-SDAG-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v3, v2, v3
-; GFX9_4-SDAG-F32DENORM-NEXT: v_add_f16_e32 v2, v3, v5
+; GFX9_4-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_f16 v2, v2, v3, v5
; GFX9_4-SDAG-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v3
+; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v6
; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX9_4-SDAG-F32DENORM-NEXT: v_mov_b32_e32 v1, v2
; GFX9_4-SDAG-F32DENORM-NEXT: s_setpc_b64 s[30:31]
@@ -2665,10 +2659,10 @@ define {float, float} @fma_chain_fpext_outer_contractable(half %x, half %y, half
; GFX9_4-GISEL-F32DENORM-LABEL: fma_chain_fpext_outer_contractable:
; GFX9_4-GISEL-F32DENORM: ; %bb.0: ; %entry
; GFX9_4-GISEL-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v3, v2, v3
-; GFX9_4-GISEL-F32DENORM-NEXT: v_add_f16_e32 v2, v3, v5
+; GFX9_4-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_f16 v2, v2, v3, v5
; GFX9_4-GISEL-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v3
+; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v6
; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX9_4-GISEL-F32DENORM-NEXT: v_mov_b32_e32 v1, v2
; GFX9_4-GISEL-F32DENORM-NEXT: s_setpc_b64 s[30:31]
@@ -2677,26 +2671,26 @@ define {float, float} @fma_chain_fpext_outer_contractable(half %x, half %y, half
; GFX12_5-SDAG-F32DENORM: ; %bb.0: ; %entry
; GFX12_5-SDAG-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12_5-SDAG-F32DENORM-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-SDAG-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX12_5-SDAG-F32DENORM-NEXT: v_add_f16_e32 v1, v2, v5
-; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX12_5-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX12_5-SDAG-F32DENORM-NEXT: v_fmac_f16_e32 v5, v2, v3
+; GFX12_5-SDAG-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12_5-SDAG-F32DENORM-NEXT: v_fmac_f16_e32 v6, v0, v1
+; GFX12_5-SDAG-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX12_5-SDAG-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12_5-SDAG-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v0, v6, 1.0, v4 op_sel_hi:[1,1,0]
; GFX12_5-SDAG-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12_5-GISEL-F32DENORM-LABEL: fma_chain_fpext_outer_contractable:
; GFX12_5-GISEL-F32DENORM: ; %bb.0: ; %entry
; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-GISEL-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX12_5-GISEL-F32DENORM-NEXT: v_add_f16_e32 v1, v2, v5
-; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, 1.0, v4 op_sel_hi:[1,1,0]
+; GFX12_5-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX12_5-GISEL-F32DENORM-NEXT: v_fmac_f16_e32 v5, v2, v3
+; GFX12_5-GISEL-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12_5-GISEL-F32DENORM-NEXT: v_fmac_f16_e32 v6, v0, v1
+; GFX12_5-GISEL-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v5
; GFX12_5-GISEL-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12_5-GISEL-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v6, 1.0, v4 op_sel_hi:[1,1,0]
; GFX12_5-GISEL-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
entry:
%mul = fmul contract half %u, %v
@@ -2736,103 +2730,49 @@ define {float, float, float} @fma_chain_fpext_noncontractable(float %x, float %y
; GFX9-GISEL-NEXT: v_add_f32_e32 v1, v2, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9_4-SDAG-F32FLUSH-LABEL: fma_chain_fpext_noncontractable:
-; GFX9_4-SDAG-F32FLUSH: ; %bb.0: ; %entry
-; GFX9_4-SDAG-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_mul_f16_e32 v7, v2, v3
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_cvt_f32_f16_e32 v6, v7
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_fma_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
-; GFX9_4-SDAG-F32FLUSH-NEXT: s_nop 0
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_fma_mix_f32 v1, v7, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
-; GFX9_4-SDAG-F32FLUSH-NEXT: v_mov_b32_e32 v2, v6
-; GFX9_4-SDAG-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9_4-GISEL-F32FLUSH-LABEL: fma_chain_fpext_noncontractable:
-; GFX9_4-GISEL-F32FLUSH: ; %bb.0: ; %entry
-; GFX9_4-GISEL-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_mul_f16_e32 v7, v2, v3
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_cvt_f32_f16_e32 v6, v7
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
-; GFX9_4-GISEL-F32FLUSH-NEXT: s_nop 0
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v1, v7, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
-; GFX9_4-GISEL-F32FLUSH-NEXT: v_mov_b32_e32 v2, v6
-; GFX9_4-GISEL-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12_5-SDAG-F32FLUSH-LABEL: fma_chain_fpext_noncontractable:
-; GFX12_5-SDAG-F32FLUSH: ; %bb.0: ; %entry
-; GFX12_5-SDAG-F32FLUSH-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12_5-SDAG-F32FLUSH-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_fma_mix_f32 v4, v2, v3, v4 op_sel_hi:[1,1,0]
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-SDAG-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_fmac_f32_e32 v4, v0, v1
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX12_5-SDAG-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12_5-SDAG-F32FLUSH-NEXT: v_mov_b32_e32 v0, v4
-; GFX12_5-SDAG-F32FLUSH-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX12_5-GISEL-F32FLUSH-LABEL: fma_chain_fpext_noncontractable:
-; GFX12_5-GISEL-F32FLUSH: ; %bb.0: ; %entry
-; GFX12_5-GISEL-F32FLUSH-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12_5-GISEL-F32FLUSH-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v4, v2, v3, v4 op_sel_hi:[1,1,0]
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-GISEL-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_fmac_f32_e32 v4, v0, v1
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX12_5-GISEL-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12_5-GISEL-F32FLUSH-NEXT: v_mov_b32_e32 v0, v4
-; GFX12_5-GISEL-F32FLUSH-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX9_4-SDAG-F32DENORM-LABEL: fma_chain_fpext_noncontractable:
-; GFX9_4-SDAG-F32DENORM: ; %bb.0: ; %entry
-; GFX9_4-SDAG-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v3, v2, v3
-; GFX9_4-SDAG-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v3
-; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1]
-; GFX9_4-SDAG-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v1, v3, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX9_4-SDAG-F32DENORM-NEXT: s_setpc_b64 s[30:31]
+; GFX9_4-SDAG-LABEL: fma_chain_fpext_noncontractable:
+; GFX9_4-SDAG: ; %bb.0: ; %entry
+; GFX9_4-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9_4-SDAG-NEXT: v_mul_f16_e32 v3, v2, v3
+; GFX9_4-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v3
+; GFX9_4-SDAG-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1]
+; GFX9_4-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9_4-SDAG-NEXT: v_fma_mix_f32 v1, v3, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX9_4-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9_4-GISEL-F32DENORM-LABEL: fma_chain_fpext_noncontractable:
-; GFX9_4-GISEL-F32DENORM: ; %bb.0: ; %entry
-; GFX9_4-GISEL-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v3, v2, v3
-; GFX9_4-GISEL-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v3
-; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1]
-; GFX9_4-GISEL-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v1, v3, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX9_4-GISEL-F32DENORM-NEXT: s_setpc_b64 s[30:31]
+; GFX9_4-GISEL-LABEL: fma_chain_fpext_noncontractable:
+; GFX9_4-GISEL: ; %bb.0: ; %entry
+; GFX9_4-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9_4-GISEL-NEXT: v_mul_f16_e32 v3, v2, v3
+; GFX9_4-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v3
+; GFX9_4-GISEL-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1]
+; GFX9_4-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9_4-GISEL-NEXT: v_fma_mix_f32 v1, v3, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX9_4-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12_5-SDAG-F32DENORM-LABEL: fma_chain_fpext_noncontractable:
-; GFX12_5-SDAG-F32DENORM: ; %bb.0: ; %entry
-; GFX12_5-SDAG-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12_5-SDAG-F32DENORM-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-SDAG-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-SDAG-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
-; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX12_5-SDAG-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX12_5-SDAG-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX12_5-SDAG-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
+; GFX12_5-SDAG-LABEL: fma_chain_fpext_noncontractable:
+; GFX12_5-SDAG: ; %bb.0: ; %entry
+; GFX12_5-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12_5-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12_5-SDAG-NEXT: v_mul_f16_e32 v2, v2, v3
+; GFX12_5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12_5-SDAG-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
+; GFX12_5-SDAG-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX12_5-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX12_5-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX12_5-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12_5-GISEL-F32DENORM-LABEL: fma_chain_fpext_noncontractable:
-; GFX12_5-GISEL-F32DENORM: ; %bb.0: ; %entry
-; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-GISEL-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
-; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
-; GFX12_5-GISEL-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX12_5-GISEL-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX12_5-GISEL-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
+; GFX12_5-GISEL-LABEL: fma_chain_fpext_noncontractable:
+; GFX12_5-GISEL: ; %bb.0: ; %entry
+; GFX12_5-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12_5-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12_5-GISEL-NEXT: v_mul_f16_e32 v2, v2, v3
+; GFX12_5-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12_5-GISEL-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
+; GFX12_5-GISEL-NEXT: v_fma_mix_f32 v1, v2, 1.0, v5 op_sel_hi:[1,1,0]
+; GFX12_5-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX12_5-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX12_5-GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
%mul = fmul contract half %u, %v
%mul.ext = fpext contract half %mul to float
@@ -2881,14 +2821,14 @@ define {float, float} @fma_chain_fpext_fsub_contractable(float %x, float %y, hal
; GFX9_4-SDAG-F32FLUSH-NEXT: v_mov_b32_e32 v0, v4
; GFX9_4-SDAG-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9_4-GISEL-LABEL: fma_chain_fpext_fsub_contractable:
-; GFX9_4-GISEL: ; %bb.0: ; %entry
-; GFX9_4-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX9_4-GISEL-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
-; GFX9_4-GISEL-NEXT: v_sub_f32_e32 v0, v0, v4
-; GFX9_4-GISEL-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
-; GFX9_4-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9_4-GISEL-F32FLUSH-LABEL: fma_chain_fpext_fsub_contractable:
+; GFX9_4-GISEL-F32FLUSH: ; %bb.0: ; %entry
+; GFX9_4-GISEL-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9_4-GISEL-F32FLUSH-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX9_4-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v6 op_sel_hi:[0,0,1]
+; GFX9_4-GISEL-F32FLUSH-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX9_4-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v1, v2, v3, -v5 op_sel_hi:[1,1,0]
+; GFX9_4-GISEL-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX12_5-SDAG-F32FLUSH-LABEL: fma_chain_fpext_fsub_contractable:
; GFX12_5-SDAG-F32FLUSH: ; %bb.0: ; %entry
@@ -2901,16 +2841,16 @@ define {float, float} @fma_chain_fpext_fsub_contractable(float %x, float %y, hal
; GFX12_5-SDAG-F32FLUSH-NEXT: v_mov_b32_e32 v0, v4
; GFX12_5-SDAG-F32FLUSH-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12_5-GISEL-LABEL: fma_chain_fpext_fsub_contractable:
-; GFX12_5-GISEL: ; %bb.0: ; %entry
-; GFX12_5-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12_5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-NEXT: v_mul_f16_e32 v2, v2, v3
-; GFX12_5-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12_5-GISEL-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
-; GFX12_5-GISEL-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
-; GFX12_5-GISEL-NEXT: v_sub_f32_e32 v0, v0, v4
-; GFX12_5-GISEL-NEXT: s_set_pc_i64 s[30:31]
+; GFX12_5-GISEL-F32FLUSH-LABEL: fma_chain_fpext_fsub_contractable:
+; GFX12_5-GISEL-F32FLUSH: ; %bb.0: ; %entry
+; GFX12_5-GISEL-F32FLUSH-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12_5-GISEL-F32FLUSH-NEXT: s_wait_kmcnt 0x0
+; GFX12_5-GISEL-F32FLUSH-NEXT: v_mul_f16_e32 v6, v2, v3
+; GFX12_5-GISEL-F32FLUSH-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12_5-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v0, v0, v1, v6 op_sel_hi:[0,0,1]
+; GFX12_5-GISEL-F32FLUSH-NEXT: v_fma_mix_f32 v1, v2, v3, -v5 op_sel_hi:[1,1,0]
+; GFX12_5-GISEL-F32FLUSH-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX12_5-GISEL-F32FLUSH-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9_4-SDAG-F32DENORM-LABEL: fma_chain_fpext_fsub_contractable:
; GFX9_4-SDAG-F32DENORM: ; %bb.0: ; %entry
@@ -2921,6 +2861,15 @@ define {float, float} @fma_chain_fpext_fsub_contractable(float %x, float %y, hal
; GFX9_4-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
; GFX9_4-SDAG-F32DENORM-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9_4-GISEL-F32DENORM-LABEL: fma_chain_fpext_fsub_contractable:
+; GFX9_4-GISEL-F32DENORM: ; %bb.0: ; %entry
+; GFX9_4-GISEL-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9_4-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
+; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
+; GFX9_4-GISEL-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX9_4-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
+; GFX9_4-GISEL-F32DENORM-NEXT: s_setpc_b64 s[30:31]
+;
; GFX12_5-SDAG-F32DENORM-LABEL: fma_chain_fpext_fsub_contractable:
; GFX12_5-SDAG-F32DENORM: ; %bb.0: ; %entry
; GFX12_5-SDAG-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2931,6 +2880,17 @@ define {float, float} @fma_chain_fpext_fsub_contractable(float %x, float %y, hal
; GFX12_5-SDAG-F32DENORM-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
; GFX12_5-SDAG-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v4
; GFX12_5-SDAG-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX12_5-GISEL-F32DENORM-LABEL: fma_chain_fpext_fsub_contractable:
+; GFX12_5-GISEL-F32DENORM: ; %bb.0: ; %entry
+; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12_5-GISEL-F32DENORM-NEXT: s_wait_kmcnt 0x0
+; GFX12_5-GISEL-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
+; GFX12_5-GISEL-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
+; GFX12_5-GISEL-F32DENORM-NEXT: v_fma_mix_f32 v1, v5, -1.0, v2 op_sel_hi:[0,1,1]
+; GFX12_5-GISEL-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX12_5-GISEL-F32DENORM-NEXT: s_set_pc_i64 s[30:31]
entry:
%mul = fmul contract reassoc half %u, %v
%mul.ext = fpext contract reassoc half %mul to float
@@ -3102,14 +3062,14 @@ define {float, float} @fma_chain_fsub_contractable(float %a, float %b, float %c,
; GFX9-SDAG-F32FLUSH-NEXT: v_mad_f32 v1, v2, v3, v5
; GFX9-SDAG-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: fma_chain_fsub_contractable:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9-GISEL-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX9-GISEL-NEXT: v_add_f32_e32 v1, v2, v5
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-F32FLUSH-LABEL: fma_chain_fsub_contractable:
+; GFX9-GISEL-F32FLUSH: ; %bb.0: ; %entry
+; GFX9-GISEL-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-F32FLUSH-NEXT: v_mul_f32_e32 v6, v2, v3
+; GFX9-GISEL-F32FLUSH-NEXT: v_fma_f32 v0, v0, v1, v6
+; GFX9-GISEL-F32FLUSH-NEXT: v_sub_f32_e32 v0, v4, v0
+; GFX9-GISEL-F32FLUSH-NEXT: v_mad_f32 v1, v2, v3, v5
+; GFX9-GISEL-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9_4-SDAG-LABEL: fma_chain_fsub_contractable:
; GFX9_4-SDAG: ; %bb.0: ; %entry
@@ -3122,10 +3082,10 @@ define {float, float} @fma_chain_fsub_contractable(float %a, float %b, float %c,
; GFX9_4-GISEL-LABEL: fma_chain_fsub_contractable:
; GFX9_4-GISEL: ; %bb.0: ; %entry
; GFX9_4-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9_4-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9_4-GISEL-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX9_4-GISEL-NEXT: v_add_f32_e32 v1, v2, v5
+; GFX9_4-GISEL-NEXT: v_mul_f32_e32 v6, v2, v3
+; GFX9_4-GISEL-NEXT: v_fmac_f32_e32 v6, v0, v1
+; GFX9_4-GISEL-NEXT: v_sub_f32_e32 v0, v4, v6
+; GFX9_4-GISEL-NEXT: v_fma_f32 v1, v2, v3, v5
; GFX9_4-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12_5-SDAG-LABEL: fma_chain_fsub_contractable:
@@ -3141,10 +3101,10 @@ define {float, float} @fma_chain_fsub_contractable(float %a, float %b, float %c,
; GFX12_5-GISEL: ; %bb.0: ; %entry
; GFX12_5-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12_5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX12_5-GISEL-NEXT: v_mul_f32_e32 v6, v2, v3
; GFX12_5-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12_5-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX12_5-GISEL-NEXT: v_dual_sub_f32 v0, v4, v0 :: v_dual_add_f32 v1, v2, v5
+; GFX12_5-GISEL-NEXT: v_dual_fmac_f32 v6, v0, v1 :: v_dual_fma_f32 v1, v2, v3, v5
+; GFX12_5-GISEL-NEXT: v_sub_f32_e32 v0, v4, v6
; GFX12_5-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9-SDAG-F32DENORM-LABEL: fma_chain_fsub_contractable:
@@ -3154,6 +3114,15 @@ define {float, float} @fma_chain_fsub_contractable(float %a, float %b, float %c,
; GFX9-SDAG-F32DENORM-NEXT: v_fma_f32 v0, -v0, v1, v4
; GFX9-SDAG-F32DENORM-NEXT: v_fma_f32 v1, v2, v3, v5
; GFX9-SDAG-F32DENORM-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-F32DENORM-LABEL: fma_chain_fsub_contractable:
+; GFX9-GISEL-F32DENORM: ; %bb.0: ; %entry
+; GFX9-GISEL-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-F32DENORM-NEXT: v_mul_f32_e32 v6, v2, v3
+; GFX9-GISEL-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v6
+; GFX9-GISEL-F32DENORM-NEXT: v_sub_f32_e32 v0, v4, v0
+; GFX9-GISEL-F32DENORM-NEXT: v_fma_f32 v1, v2, v3, v5
+; GFX9-GISEL-F32DENORM-NEXT: s_setpc_b64 s[30:31]
entry:
%mul = fmul contract reassoc nsz float %c, %d
%fma.res = call contract reassoc nsz float @llvm.fma.f32(float %a, float %b, float %mul)
@@ -3257,61 +3226,77 @@ entry:
; reassociation does not fire and the multiply remains.
; Expected: fma chain (no v_mul) on denorm-capable targets; v_mul remains on gfx9-flush.
define {float, float} @fma_chain_fadd_reassoc(float %a, float %b, float %c, float %d, float %e, float %f) #0 {
-; GFX9-SDAG-LABEL: fma_chain_fadd_reassoc:
-; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9-SDAG-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9-SDAG-NEXT: v_add_f32_e32 v1, v2, v5
-; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-F32FLUSH-LABEL: fma_chain_fadd_reassoc:
+; GFX9-SDAG-F32FLUSH: ; %bb.0: ; %entry
+; GFX9-SDAG-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-F32FLUSH-NEXT: v_mul_f32_e32 v6, v2, v3
+; GFX9-SDAG-F32FLUSH-NEXT: v_fma_f32 v0, v0, v1, v6
+; GFX9-SDAG-F32FLUSH-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9-SDAG-F32FLUSH-NEXT: v_mad_f32 v1, v2, v3, v5
+; GFX9-SDAG-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: fma_chain_fadd_reassoc:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9-GISEL-NEXT: v_add_f32_e32 v1, v2, v5
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-GISEL-F32FLUSH-LABEL: fma_chain_fadd_reassoc:
+; GFX9-GISEL-F32FLUSH: ; %bb.0: ; %entry
+; GFX9-GISEL-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-F32FLUSH-NEXT: v_mul_f32_e32 v6, v2, v3
+; GFX9-GISEL-F32FLUSH-NEXT: v_fma_f32 v0, v0, v1, v6
+; GFX9-GISEL-F32FLUSH-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9-GISEL-F32FLUSH-NEXT: v_mad_f32 v1, v2, v3, v5
+; GFX9-GISEL-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9_4-SDAG-LABEL: fma_chain_fadd_reassoc:
; GFX9_4-SDAG: ; %bb.0: ; %entry
; GFX9_4-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-SDAG-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9_4-SDAG-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9_4-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9_4-SDAG-NEXT: v_add_f32_e32 v1, v2, v5
+; GFX9_4-SDAG-NEXT: v_fma_f32 v4, v2, v3, v4
+; GFX9_4-SDAG-NEXT: v_fmac_f32_e32 v4, v0, v1
+; GFX9_4-SDAG-NEXT: v_fma_f32 v1, v2, v3, v5
+; GFX9_4-SDAG-NEXT: v_mov_b32_e32 v0, v4
; GFX9_4-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9_4-GISEL-LABEL: fma_chain_fadd_reassoc:
; GFX9_4-GISEL: ; %bb.0: ; %entry
; GFX9_4-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9_4-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
-; GFX9_4-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9_4-GISEL-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX9_4-GISEL-NEXT: v_add_f32_e32 v1, v2, v5
+; GFX9_4-GISEL-NEXT: v_fma_f32 v4, v2, v3, v4
+; GFX9_4-GISEL-NEXT: v_fmac_f32_e32 v4, v0, v1
+; GFX9_4-GISEL-NEXT: v_fma_f32 v1, v2, v3, v5
+; GFX9_4-GISEL-NEXT: v_mov_b32_e32 v0, v4
; GFX9_4-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12_5-SDAG-LABEL: fma_chain_fadd_reassoc:
; GFX12_5-SDAG: ; %bb.0: ; %entry
; GFX12_5-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12_5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-SDAG-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX12_5-SDAG-NEXT: v_fma_f32 v4, v2, v3, v4
; GFX12_5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12_5-SDAG-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX12_5-SDAG-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v2, v5
+; GFX12_5-SDAG-NEXT: v_dual_fmac_f32 v4, v0, v1 :: v_dual_fma_f32 v1, v2, v3, v5
+; GFX12_5-SDAG-NEXT: v_mov_b32_e32 v0, v4
; GFX12_5-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12_5-GISEL-LABEL: fma_chain_fadd_reassoc:
; GFX12_5-GISEL: ; %bb.0: ; %entry
; GFX12_5-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12_5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12_5-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX12_5-GISEL-NEXT: v_fma_f32 v4, v2, v3, v4
; GFX12_5-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12_5-GISEL-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX12_5-GISEL-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v2, v5
+; GFX12_5-GISEL-NEXT: v_dual_fmac_f32 v4, v0, v1 :: v_dual_fma_f32 v1, v2, v3, v5
+; GFX12_5-GISEL-NEXT: v_mov_b32_e32 v0, v4
; GFX12_5-GISEL-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX9-SDAG-F32DENORM-LABEL: fma_chain_fadd_reassoc:
+; GFX9-SDAG-F32DENORM: ; %bb.0: ; %entry
+; GFX9-SDAG-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-F32DENORM-NEXT: v_fma_f32 v4, v2, v3, v4
+; GFX9-SDAG-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v4
+; GFX9-SDAG-F32DENORM-NEXT: v_fma_f32 v1, v2, v3, v5
+; GFX9-SDAG-F32DENORM-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-F32DENORM-LABEL: fma_chain_fadd_reassoc:
+; GFX9-GISEL-F32DENORM: ; %bb.0: ; %entry
+; GFX9-GISEL-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-F32DENORM-NEXT: v_fma_f32 v4, v2, v3, v4
+; GFX9-GISEL-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v4
+; GFX9-GISEL-F32DENORM-NEXT: v_fma_f32 v1, v2, v3, v5
+; GFX9-GISEL-F32DENORM-NEXT: s_setpc_b64 s[30:31]
entry:
%mul = fmul contract reassoc nsz float %c, %d
%fma.res = call contract reassoc nsz float @llvm.fma.f32(float %a, float %b, float %mul)
More information about the llvm-branch-commits
mailing list