[llvm] [AMDGPU] Gate fneg folding into neg modifier with NaN check. (PR #201440)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 12:59:49 PDT 2026
https://github.com/carlobertolli created https://github.com/llvm/llvm-project/pull/201440
The optimization in question folds
fneg(fmul(x,y))->fmul(x, NEG(y))
If x turns out to be a +NaN at runtime, fmul propagates it, including its sign, regardless of the NEG(y). That means that we return +NaN instead of the expected -NaN. This patch gates such folding for all instructions for which it is allowed to only cases where NaN is proved not to be present or nnan attribute is present.
Assisted-by: Claude Opus
>From 7e65ab5408a98590aeb80af1027396c71d089136 Mon Sep 17 00:00:00 2001
From: Carlo Bertolli <carlo.bertolli at amd.com>
Date: Wed, 3 Jun 2026 19:52:28 +0000
Subject: [PATCH] [AMDGPU] Gate fneg folding into neg modifier with NaN check.
The optimization in question folds fneg(fmul(x,y))->fmul(x, NEG(y)) If x
turns out to be a +NaN at runtime, fmul propagates it, including its sign,
regardless of the NEG(y). That means that we return +NaN instead of the
expected -NaN. This patch gates such folding for all instructions for which
it is allowed to only cases where NaN is proved not to be present or nnan
attribute is present.
Assisted-by: Claude Opus
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 72 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
.../CodeGen/AMDGPU/dagcombine-fma-crash.ll | 19 +-
.../CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll | 22 +-
llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll | 134 +-
llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll | 143 +-
llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll | 452 ++--
.../CodeGen/AMDGPU/fneg-combines.legal.f16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fneg-combines.ll | 226 +-
llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll | 116 +-
.../test/CodeGen/AMDGPU/fneg-nan-sign-fold.ll | 149 ++
.../AMDGPU/fneg-nan-sign-preservation.ll | 198 ++
llvm/test/CodeGen/AMDGPU/fpext-free.ll | 38 +-
llvm/test/CodeGen/AMDGPU/mad-combine.ll | 16 +-
llvm/test/CodeGen/AMDGPU/mad-mix.ll | 136 +-
.../AMDGPU/select-fabs-fneg-extract-legacy.ll | 45 +-
.../AMDGPU/select-fabs-fneg-extract.ll | 2239 ++++++++++++++---
.../AMDGPU/select-fabs-fneg-extract.v2f16.ll | 38 +-
.../CodeGen/AMDGPU/setcc-fneg-constant.ll | 323 ++-
llvm/test/CodeGen/AMDGPU/v_mac.ll | 633 ++++-
llvm/test/CodeGen/AMDGPU/v_mac_f16.ll | 2049 ++++++++++++---
21 files changed, 5594 insertions(+), 1470 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/fneg-nan-sign-fold.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/fneg-nan-sign-preservation.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f0918a86be757..27eba3476d445 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -931,12 +931,36 @@ SDValue AMDGPUTargetLowering::getNegatedExpression(
SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
NegatibleCost &Cost, unsigned Depth) const {
+ // Multiply instructions propagate NaN from the NaN-carrying source
+ // operand, taking the sign from that operand (including any NEG modifier
+ // applied to it). A NEG modifier on the OTHER (non-NaN) operand does NOT
+ // affect the result NaN sign. This means fneg(fmul(x,y)) cannot be folded
+ // to fmul(x,fneg(y)) when x might be NaN, because the outer fneg would
+ // flip x's NaN sign but the inner NEG on y doesn't. The fold is safe when
+ // the non-negated operand is known never-NaN, or the operation has nnan.
+ //
+ // For FMA/FMAD, the same constraint applies to the multiply operands.
+ // Negating the addend is always NaN-sign-safe.
+
switch (Op.getOpcode()) {
case ISD::FMA:
- case ISD::FMAD: {
+ case ISD::FMULADD:
+ case ISD::FMAD:
// Negating a fma is not free if it has users without source mods.
if (!allUsesHaveSourceMods(Op.getNode()))
return SDValue();
+ [[fallthrough]];
+ case ISD::FMUL:
+ case ISD::FDIV: {
+ // Block the fold when neither multiply operand is known non-NaN, unless
+ // nnan is set. Whichever operand receives the NEG modifier is only
+ // correct when the OTHER operand is known non-NaN; otherwise a runtime
+ // NaN in that other operand would propagate with the wrong sign.
+ if (!Op->getFlags().hasNoNaNs()) {
+ SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
+ if (!DAG.isKnownNeverNaN(X) || !DAG.isKnownNeverNaN(Y))
+ return SDValue();
+ }
break;
}
case AMDGPUISD::RCP: {
@@ -5292,12 +5316,30 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
SDValue LHS = N0.getOperand(0);
SDValue RHS = N0.getOperand(1);
- if (LHS.getOpcode() == ISD::FNEG)
+ // Multiply NaN propagation: when one source is NaN, the hardware
+ // takes the NaN sign from that source operand (after its NEG modifier).
+ // A NEG on the non-NaN operand doesn't affect the result NaN sign.
+ // Only fold if we know that the non negated operand is known never-NAN or
+ // the operation has nnan.
+ bool CanFold = N0->getFlags().hasNoNaNs();
+
+ if (LHS.getOpcode() == ISD::FNEG) {
+ if (!CanFold && !DAG.isKnownNeverNaN(RHS))
+ return SDValue();
LHS = LHS.getOperand(0);
- else if (RHS.getOpcode() == ISD::FNEG)
+ } else if (RHS.getOpcode() == ISD::FNEG) {
+ if (!CanFold && !DAG.isKnownNeverNaN(LHS))
+ return SDValue();
RHS = RHS.getOperand(0);
- else
+ } else if (CanFold) {
+ RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
+ } else if (DAG.isKnownNeverNaN(LHS)) {
RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
+ } else if (DAG.isKnownNeverNaN(RHS)) {
+ LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
+ } else {
+ return SDValue();
+ }
SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
if (Res.getOpcode() != Opc)
@@ -5317,12 +5359,28 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
SDValue MHS = N0.getOperand(1);
SDValue RHS = N0.getOperand(2);
- if (LHS.getOpcode() == ISD::FNEG)
+ // Same multiply NaN sign concern as FMUL: the operand that keeps its
+ // NEG state unchanged must be known never-NaN, or nnan must be set.
+ // Negating the addend is always NaN-sign-safe.
+ bool CanFold = N0->getFlags().hasNoNaNs();
+
+ if (LHS.getOpcode() == ISD::FNEG) {
+ if (!CanFold && !DAG.isKnownNeverNaN(MHS))
+ return SDValue();
LHS = LHS.getOperand(0);
- else if (MHS.getOpcode() == ISD::FNEG)
+ } else if (MHS.getOpcode() == ISD::FNEG) {
+ if (!CanFold && !DAG.isKnownNeverNaN(LHS))
+ return SDValue();
MHS = MHS.getOperand(0);
- else
+ } else if (CanFold) {
+ MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
+ } else if (DAG.isKnownNeverNaN(LHS)) {
MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
+ } else if (DAG.isKnownNeverNaN(MHS)) {
+ LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
+ } else {
+ return SDValue();
+ }
if (RHS.getOpcode() != ISD::FNEG)
RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 86f2479490c29..c4fbf9d72c05d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -17677,11 +17677,13 @@ SDValue SITargetLowering::performFSubCombine(SDNode *N,
if (RHS.getOpcode() == ISD::FADD) {
// (fsub c, (fadd a, a)) -> mad -2.0, a, c
+ // Bail out if a is NaN, as mad would not propagate sign from -2.0.
SDValue A = RHS.getOperand(0);
if (A == RHS.getOperand(1)) {
unsigned FusedOp = getFusedOpcode(DAG, N, RHS.getNode());
- if (FusedOp != 0) {
+ if (FusedOp != 0 &&
+ (N->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(A))) {
const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS);
}
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
index 142494a803755..ee146767d85d0 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
@@ -86,18 +86,17 @@ define float @test2(float %arg, float %arg1) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1082130432, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 1120534528
- ; CHECK-NEXT: [[V_FMAC_F32_e64_:%[0-9]+]]:vgpr_32 = nsz contract reassoc nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, killed [[S_MOV_B32_]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_:%[0-9]+]]:vgpr_32 = nsz contract reassoc nofpexcept V_MUL_F32_e64 0, [[COPY]], 0, killed [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; CHECK-NEXT: [[V_FMAC_F32_e64_1:%[0-9]+]]:vgpr_32 = nsz contract nofpexcept V_FMAC_F32_e64 0, [[COPY1]], 0, killed [[S_MOV_B32_1]], 0, [[V_FMAC_F32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nsz contract reassoc nofpexcept V_ADD_F32_e64 0, [[V_FMAC_F32_e64_1]], 0, [[V_MOV_B32_e32_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_RCP_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RCP_F32_e64 0, [[V_FMAC_F32_e64_1]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_RCP_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_RCP_F32_e64 0, killed [[V_ADD_F32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MUL_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[V_RCP_F32_e64_1]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MAX_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_MAX_F32_e64 0, killed [[V_MUL_F32_e64_]], 0, [[V_RCP_F32_e64_1]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_MAX_F32_e64_]], 0, killed [[V_RCP_F32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_F32_e64_1]]
+ ; CHECK-NEXT: [[V_FMAC_F32_e64_:%[0-9]+]]:vgpr_32 = nsz contract nofpexcept V_FMAC_F32_e64 0, [[COPY1]], 0, killed [[S_MOV_B32_1]], 0, [[V_MUL_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 1065353216
+ ; CHECK-NEXT: [[V_SUB_F32_e64_:%[0-9]+]]:vgpr_32 = nsz contract reassoc nofpexcept V_SUB_F32_e64 0, killed [[V_FMAC_F32_e64_]], 0, killed [[S_MOV_B32_2]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_RCP_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RCP_F32_e64 0, killed [[V_SUB_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[V_RCP_F32_e64_]], 0, [[COPY1]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MAX_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_MAX_F32_e64 0, killed [[V_MUL_F32_e64_1]], 0, [[V_RCP_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_MAX_F32_e64_]], 0, [[V_RCP_F32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_F32_e64_]]
; CHECK-NEXT: SI_RETURN implicit $vgpr0
bb:
%i = fmul contract float %arg1, 1.000000e+02
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index da7cae86cf1ea..f7980491f2158 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -435,27 +435,25 @@ define amdgpu_kernel void @div_v4_c_by_minus_x_25ulp(ptr addrspace(1) %arg) {
; GCN-FLUSH: ; %bb.0:
; GCN-FLUSH-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GCN-FLUSH-NEXT: v_mov_b32_e32 v0, 0x6f800000
-; GCN-FLUSH-NEXT: v_mov_b32_e32 v2, 0x2f800000
+; GCN-FLUSH-NEXT: v_mov_b32_e32 v1, 0x2f800000
; GCN-FLUSH-NEXT: v_mov_b32_e32 v4, 0
; GCN-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
; GCN-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
; GCN-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
; GCN-FLUSH-NEXT: v_cmp_gt_f32_e64 vcc, |s0|, v0
-; GCN-FLUSH-NEXT: v_cndmask_b32_e32 v3, 1.0, v2, vcc
+; GCN-FLUSH-NEXT: v_cndmask_b32_e32 v3, 1.0, v1, vcc
; GCN-FLUSH-NEXT: v_cmp_gt_f32_e64 vcc, |s3|, v0
-; GCN-FLUSH-NEXT: v_mul_f32_e32 v5, s0, v3
-; GCN-FLUSH-NEXT: v_mul_f32_e64 v6, -s0, v3
-; GCN-FLUSH-NEXT: v_cndmask_b32_e32 v7, 1.0, v2, vcc
-; GCN-FLUSH-NEXT: v_rcp_f32_e32 v5, v5
-; GCN-FLUSH-NEXT: v_rcp_f32_e32 v6, v6
-; GCN-FLUSH-NEXT: v_mul_f32_e32 v0, s3, v7
-; GCN-FLUSH-NEXT: v_rcp_f32_e32 v8, v0
+; GCN-FLUSH-NEXT: v_cndmask_b32_e32 v5, 1.0, v1, vcc
+; GCN-FLUSH-NEXT: v_mul_f32_e64 v0, -s0, v3
+; GCN-FLUSH-NEXT: v_mul_f32_e64 v1, -s3, v5
+; GCN-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
+; GCN-FLUSH-NEXT: v_rcp_f32_e32 v6, v1
; GCN-FLUSH-NEXT: v_rcp_f32_e64 v1, -s1
; GCN-FLUSH-NEXT: v_rcp_f32_e32 v2, s2
-; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v6, v5
+; GCN-FLUSH-NEXT: v_add_f32_e32 v0, v0, v0
+; GCN-FLUSH-NEXT: v_mul_f32_e32 v6, -2.0, v6
; GCN-FLUSH-NEXT: v_mul_f32_e32 v0, v3, v0
-; GCN-FLUSH-NEXT: v_add_f32_e32 v3, v8, v8
-; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v7, v3
+; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v5, v6
; GCN-FLUSH-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GCN-FLUSH-NEXT: s_endpgm
%load = load <4 x float>, ptr addrspace(1) %arg, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 51b6d17312ed7..1f14cfe0dd4b0 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -1291,7 +1291,8 @@ define amdgpu_kernel void @fmuladd_neg_2.0_a_b_f16(ptr addrspace(1) %out, ptr ad
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: flat_load_ushort v2, v[2:3] glc
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mac_f16_e32 v2, -2.0, v4
+; VI-FLUSH-NEXT: v_add_f16_e32 v3, v4, v4
+; VI-FLUSH-NEXT: v_sub_f16_e32 v2, v2, v3
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_endpgm
;
@@ -1461,7 +1462,8 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(ptr addrspace(1) %out, pt
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: flat_load_ushort v2, v[2:3] glc
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mac_f16_e32 v2, 2.0, v4
+; VI-FLUSH-NEXT: v_sub_f16_e64 v3, -v4, v4
+; VI-FLUSH-NEXT: v_sub_f16_e32 v2, v2, v3
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_endpgm
;
@@ -1492,8 +1494,8 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(ptr addrspace(1) %out, pt
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[0:1] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v1, v1
-; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v2, v1
+; GFX10-FLUSH-NEXT: v_sub_f16_e64 v1, -v1, v1
+; GFX10-FLUSH-NEXT: v_sub_f16_e32 v1, v2, v1
; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-FLUSH-NEXT: s_endpgm
;
@@ -1521,9 +1523,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(ptr addrspace(1) %out, pt
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[0:1] offset:2 glc dlc
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v0.l
+; GFX11-FLUSH-TRUE16-NEXT: v_sub_f16_e64 v0.l, -v0.l, v0.l
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v0.l
+; GFX11-FLUSH-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.h, v0.l
; GFX11-FLUSH-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-FLUSH-TRUE16-NEXT: s_endpgm
;
@@ -1538,9 +1540,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(ptr addrspace(1) %out, pt
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: global_load_u16 v2, v0, s[0:1] offset:2 glc dlc
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v1, v1
+; GFX11-FLUSH-FAKE16-NEXT: v_sub_f16_e64 v1, -v1, v1
; GFX11-FLUSH-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v2, v1
+; GFX11-FLUSH-FAKE16-NEXT: v_sub_f16_e32 v1, v2, v1
; GFX11-FLUSH-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FLUSH-FAKE16-NEXT: s_endpgm
;
@@ -1633,7 +1635,8 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(ptr addrspace(1) %out, ptr ad
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: flat_load_ushort v2, v[2:3] glc
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mac_f16_e32 v2, -2.0, v4
+; VI-FLUSH-NEXT: v_sub_f16_e64 v3, -v4, v4
+; VI-FLUSH-NEXT: v_add_f16_e32 v2, v3, v2
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_endpgm
;
@@ -1664,8 +1667,8 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(ptr addrspace(1) %out, ptr ad
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[0:1] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v1, v1
-; GFX10-FLUSH-NEXT: v_sub_f16_e32 v1, v2, v1
+; GFX10-FLUSH-NEXT: v_sub_f16_e64 v1, -v1, v1
+; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-FLUSH-NEXT: s_endpgm
;
@@ -1693,9 +1696,9 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(ptr addrspace(1) %out, ptr ad
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[0:1] offset:2 glc dlc
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v0.l
+; GFX11-FLUSH-TRUE16-NEXT: v_sub_f16_e64 v0.l, -v0.l, v0.l
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.h, v0.l
+; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-FLUSH-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-FLUSH-TRUE16-NEXT: s_endpgm
;
@@ -1710,9 +1713,9 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(ptr addrspace(1) %out, ptr ad
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: global_load_u16 v2, v0, s[0:1] offset:2 glc dlc
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v1, v1
+; GFX11-FLUSH-FAKE16-NEXT: v_sub_f16_e64 v1, -v1, v1
; GFX11-FLUSH-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-FAKE16-NEXT: v_sub_f16_e32 v1, v2, v1
+; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v1, v2
; GFX11-FLUSH-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FLUSH-FAKE16-NEXT: s_endpgm
;
@@ -2923,7 +2926,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
-; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v3, v1
+; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v1, v3
; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-FLUSH-NEXT: s_endpgm
;
@@ -2939,7 +2942,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
-; GFX10-DENORM-STRICT-NEXT: v_add_f16_e32 v1, v3, v1
+; GFX10-DENORM-STRICT-NEXT: v_add_f16_e32 v1, v1, v3
; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
@@ -2973,7 +2976,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.l, v0.l
+; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-FLUSH-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX11-FLUSH-TRUE16-NEXT: s_endpgm
;
@@ -2992,7 +2995,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX11-FLUSH-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v3, v1
+; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e32 v1, v1, v3
; GFX11-FLUSH-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FLUSH-FAKE16-NEXT: s_endpgm
;
@@ -3011,7 +3014,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX11-DENORM-STRICT-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-STRICT-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-DENORM-STRICT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-DENORM-STRICT-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.l, v0.l
+; GFX11-DENORM-STRICT-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-DENORM-STRICT-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX11-DENORM-STRICT-TRUE16-NEXT: s_endpgm
;
@@ -3030,7 +3033,7 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
; GFX11-DENORM-STRICT-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-STRICT-FAKE16-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX11-DENORM-STRICT-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-DENORM-STRICT-FAKE16-NEXT: v_add_f16_e32 v1, v3, v1
+; GFX11-DENORM-STRICT-FAKE16-NEXT: v_add_f16_e32 v1, v1, v3
; GFX11-DENORM-STRICT-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-DENORM-STRICT-FAKE16-NEXT: s_endpgm
;
@@ -3326,27 +3329,29 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f16(ptr addrspace(1) %out, ptr addrsp
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: flat_load_ushort v2, v[2:3] glc
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mac_f16_e32 v2, -2.0, v4
+; VI-FLUSH-NEXT: v_add_f16_e32 v3, v4, v4
+; VI-FLUSH-NEXT: v_sub_f16_e32 v2, v2, v3
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_endpgm
;
-; VI-DENORM-CONTRACT-LABEL: fsub_c_fadd_a_a_f16:
-; VI-DENORM-CONTRACT: ; %bb.0:
-; VI-DENORM-CONTRACT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; VI-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DENORM-CONTRACT-NEXT: v_mov_b32_e32 v1, s1
-; VI-DENORM-CONTRACT-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; VI-DENORM-CONTRACT-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-DENORM-CONTRACT-NEXT: v_add_u32_e32 v2, vcc, 2, v0
-; VI-DENORM-CONTRACT-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; VI-DENORM-CONTRACT-NEXT: flat_load_ushort v4, v[0:1] glc
-; VI-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; VI-DENORM-CONTRACT-NEXT: flat_load_ushort v2, v[2:3] glc
-; VI-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; VI-DENORM-CONTRACT-NEXT: v_fma_f16 v2, v4, -2.0, v2
-; VI-DENORM-CONTRACT-NEXT: flat_store_short v[0:1], v2
-; VI-DENORM-CONTRACT-NEXT: s_endpgm
+; VI-DENORM-LABEL: fsub_c_fadd_a_a_f16:
+; VI-DENORM: ; %bb.0:
+; VI-DENORM-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-DENORM-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; VI-DENORM-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
+; VI-DENORM-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; VI-DENORM-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-DENORM-NEXT: v_add_u32_e32 v2, vcc, 2, v0
+; VI-DENORM-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-DENORM-NEXT: flat_load_ushort v4, v[0:1] glc
+; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
+; VI-DENORM-NEXT: flat_load_ushort v2, v[2:3] glc
+; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
+; VI-DENORM-NEXT: v_add_f16_e32 v3, v4, v4
+; VI-DENORM-NEXT: v_sub_f16_e32 v2, v2, v3
+; VI-DENORM-NEXT: flat_store_short v[0:1], v2
+; VI-DENORM-NEXT: s_endpgm
;
; GFX10-FLUSH-LABEL: fsub_c_fadd_a_a_f16:
; GFX10-FLUSH: ; %bb.0:
@@ -3362,32 +3367,19 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f16(ptr addrspace(1) %out, ptr addrsp
; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-FLUSH-NEXT: s_endpgm
;
-; GFX10-DENORM-STRICT-LABEL: fsub_c_fadd_a_a_f16:
-; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[0:1] glc dlc
-; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[0:1] offset:2 glc dlc
-; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: v_add_f16_e32 v1, v1, v1
-; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e32 v1, v2, v1
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
-; GFX10-DENORM-STRICT-NEXT: s_endpgm
-;
-; GFX10-DENORM-CONTRACT-LABEL: fsub_c_fadd_a_a_f16:
-; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
-; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[0:1] glc dlc
-; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[0:1] offset:2 glc dlc
-; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: v_fmac_f16_e32 v2, -2.0, v1
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v2, s[0:1]
-; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
+; GFX10-DENORM-LABEL: fsub_c_fadd_a_a_f16:
+; GFX10-DENORM: ; %bb.0:
+; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-DENORM-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-DENORM-NEXT: global_load_ushort v1, v0, s[0:1] glc dlc
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DENORM-NEXT: global_load_ushort v2, v0, s[0:1] offset:2 glc dlc
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DENORM-NEXT: v_add_f16_e32 v1, v1, v1
+; GFX10-DENORM-NEXT: v_sub_f16_e32 v1, v2, v1
+; GFX10-DENORM-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-NEXT: s_endpgm
;
; GFX11-FLUSH-TRUE16-LABEL: fsub_c_fadd_a_a_f16:
; GFX11-FLUSH-TRUE16: ; %bb.0:
@@ -3468,8 +3460,10 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f16(ptr addrspace(1) %out, ptr addrsp
; GFX11-DENORM-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-CONTRACT-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[0:1] offset:2 glc dlc
; GFX11-DENORM-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DENORM-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v0.h, -2.0, v0.l
-; GFX11-DENORM-CONTRACT-TRUE16-NEXT: global_store_d16_hi_b16 v1, v0, s[0:1]
+; GFX11-DENORM-CONTRACT-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v0.l
+; GFX11-DENORM-CONTRACT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-CONTRACT-TRUE16-NEXT: v_sub_f16_e32 v0.l, v0.h, v0.l
+; GFX11-DENORM-CONTRACT-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-DENORM-CONTRACT-TRUE16-NEXT: s_endpgm
;
; GFX11-DENORM-CONTRACT-FAKE16-LABEL: fsub_c_fadd_a_a_f16:
@@ -3483,8 +3477,10 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f16(ptr addrspace(1) %out, ptr addrsp
; GFX11-DENORM-CONTRACT-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-CONTRACT-FAKE16-NEXT: global_load_u16 v2, v0, s[0:1] offset:2 glc dlc
; GFX11-DENORM-CONTRACT-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DENORM-CONTRACT-FAKE16-NEXT: v_fmac_f16_e32 v2, -2.0, v1
-; GFX11-DENORM-CONTRACT-FAKE16-NEXT: global_store_b16 v0, v2, s[0:1]
+; GFX11-DENORM-CONTRACT-FAKE16-NEXT: v_add_f16_e32 v1, v1, v1
+; GFX11-DENORM-CONTRACT-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-CONTRACT-FAKE16-NEXT: v_sub_f16_e32 v1, v2, v1
+; GFX11-DENORM-CONTRACT-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-DENORM-CONTRACT-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep.0 = getelementptr half, ptr addrspace(1) %out, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
index cbda0621a3181..5f13b3748673b 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
@@ -1090,8 +1090,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_a_b_f32(ptr addrspace(1) %out, ptr ad
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; SI-FLUSH-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; SI-FLUSH-NEXT: v_mac_f32_e32 v3, -2.0, v2
-; SI-FLUSH-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-FLUSH-NEXT: v_add_f32_e32 v2, v2, v2
+; SI-FLUSH-NEXT: v_sub_f32_e32 v2, v3, v2
+; SI-FLUSH-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-FLUSH-NEXT: s_endpgm
;
; SI-DENORM-FASTFMA-LABEL: fmuladd_neg_2.0_a_b_f32:
@@ -1136,8 +1137,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_a_b_f32(ptr addrspace(1) %out, ptr ad
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLUSH-MAD-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLUSH-MAD-NEXT: v_mac_f32_e32 v2, -2.0, v1
-; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-FLUSH-MAD-NEXT: v_add_f32_e32 v1, v1, v1
+; GFX9-FLUSH-MAD-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-FLUSH-MAD-NEXT: s_endpgm
;
; GFX9-DENORM-FASTFMA-MAD-LABEL: fmuladd_neg_2.0_a_b_f32:
@@ -1217,8 +1219,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f32(ptr addrspace(1) %out, pt
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; SI-FLUSH-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; SI-FLUSH-NEXT: v_mac_f32_e32 v3, 2.0, v2
-; SI-FLUSH-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-FLUSH-NEXT: v_sub_f32_e64 v2, -v2, v2
+; SI-FLUSH-NEXT: v_sub_f32_e32 v2, v3, v2
+; SI-FLUSH-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-FLUSH-NEXT: s_endpgm
;
; SI-DENORM-FASTFMA-LABEL: fmuladd_neg_2.0_neg_a_b_f32:
@@ -1249,8 +1252,8 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f32(ptr addrspace(1) %out, pt
; SI-DENORM-SLOWFMA-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMA-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-DENORM-SLOWFMA-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v2, v2
-; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v3, v2
+; SI-DENORM-SLOWFMA-NEXT: v_sub_f32_e64 v2, -v2, v2
+; SI-DENORM-SLOWFMA-NEXT: v_sub_f32_e32 v2, v3, v2
; SI-DENORM-SLOWFMA-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMA-NEXT: s_endpgm
;
@@ -1263,8 +1266,9 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f32(ptr addrspace(1) %out, pt
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLUSH-MAD-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLUSH-MAD-NEXT: v_mac_f32_e32 v2, 2.0, v1
-; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-FLUSH-MAD-NEXT: v_sub_f32_e64 v1, -v1, v1
+; GFX9-FLUSH-MAD-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-FLUSH-MAD-NEXT: s_endpgm
;
; GFX9-DENORM-FASTFMA-MAD-LABEL: fmuladd_neg_2.0_neg_a_b_f32:
@@ -1346,8 +1350,9 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr ad
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; SI-FLUSH-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; SI-FLUSH-NEXT: v_mac_f32_e32 v3, -2.0, v2
-; SI-FLUSH-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-FLUSH-NEXT: v_sub_f32_e64 v2, -v2, v2
+; SI-FLUSH-NEXT: v_add_f32_e32 v2, v2, v3
+; SI-FLUSH-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-FLUSH-NEXT: s_endpgm
;
; SI-DENORM-FASTFMA-LABEL: fmuladd_2.0_neg_a_b_f32:
@@ -1378,8 +1383,8 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr ad
; SI-DENORM-SLOWFMA-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMA-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-DENORM-SLOWFMA-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v2, v2
-; SI-DENORM-SLOWFMA-NEXT: v_sub_f32_e32 v2, v3, v2
+; SI-DENORM-SLOWFMA-NEXT: v_sub_f32_e64 v2, -v2, v2
+; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v2, v3
; SI-DENORM-SLOWFMA-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMA-NEXT: s_endpgm
;
@@ -1392,8 +1397,9 @@ define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr ad
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLUSH-MAD-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc
; GFX9-FLUSH-MAD-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLUSH-MAD-NEXT: v_mac_f32_e32 v2, -2.0, v1
-; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-FLUSH-MAD-NEXT: v_sub_f32_e64 v1, -v1, v1
+; GFX9-FLUSH-MAD-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX9-FLUSH-MAD-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-FLUSH-MAD-NEXT: s_endpgm
;
; GFX9-DENORM-FASTFMA-MAD-LABEL: fmuladd_2.0_neg_a_b_f32:
@@ -2262,7 +2268,7 @@ define amdgpu_kernel void @neg_neg_mad_f32(ptr addrspace(1) noalias nocapture %o
; SI-DENORM-FASTFMA-STRICT-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-FASTFMA-STRICT-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-DENORM-FASTFMA-STRICT-NEXT: v_mul_f32_e32 v2, v2, v3
-; SI-DENORM-FASTFMA-STRICT-NEXT: v_add_f32_e32 v2, v4, v2
+; SI-DENORM-FASTFMA-STRICT-NEXT: v_add_f32_e32 v2, v2, v4
; SI-DENORM-FASTFMA-STRICT-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-FASTFMA-STRICT-NEXT: s_endpgm
;
@@ -2283,7 +2289,7 @@ define amdgpu_kernel void @neg_neg_mad_f32(ptr addrspace(1) noalias nocapture %o
; SI-DENORM-SLOWFMA-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMA-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-DENORM-SLOWFMA-NEXT: v_mul_f32_e32 v2, v2, v3
-; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v4, v2
+; SI-DENORM-SLOWFMA-NEXT: v_add_f32_e32 v2, v2, v4
; SI-DENORM-SLOWFMA-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMA-NEXT: s_endpgm
;
@@ -2334,7 +2340,7 @@ define amdgpu_kernel void @neg_neg_mad_f32(ptr addrspace(1) noalias nocapture %o
; GFX9-DENORM-NEXT: global_load_dword v3, v0, s[2:3] offset:8 glc
; GFX9-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX9-DENORM-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX9-DENORM-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX9-DENORM-NEXT: v_add_f32_e32 v1, v1, v3
; GFX9-DENORM-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-DENORM-NEXT: s_endpgm
;
@@ -2365,7 +2371,7 @@ define amdgpu_kernel void @neg_neg_mad_f32(ptr addrspace(1) noalias nocapture %o
; GFX10-DENORM-NEXT: global_load_dword v3, v0, s[2:3] offset:8 glc dlc
; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX10-DENORM-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX10-DENORM-NEXT: v_add_f32_e32 v1, v1, v3
; GFX10-DENORM-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-DENORM-NEXT: s_endpgm
%tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
@@ -2562,26 +2568,27 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f32(ptr addrspace(1) %out, ptr addrsp
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; SI-FLUSH-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
; SI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; SI-FLUSH-NEXT: v_mac_f32_e32 v3, -2.0, v2
-; SI-FLUSH-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-FLUSH-NEXT: v_add_f32_e32 v2, v2, v2
+; SI-FLUSH-NEXT: v_sub_f32_e32 v2, v3, v2
+; SI-FLUSH-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-FLUSH-NEXT: s_endpgm
;
-; SI-DENORM-FASTFMA-STRICT-LABEL: fsub_c_fadd_a_a_f32:
-; SI-DENORM-FASTFMA-STRICT: ; %bb.0:
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_mov_b32 s3, 0xf000
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_mov_b32 s2, 0
-; SI-DENORM-FASTFMA-STRICT-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-DENORM-FASTFMA-STRICT-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; SI-DENORM-FASTFMA-STRICT-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMA-STRICT-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMA-STRICT-NEXT: v_add_f32_e32 v2, v2, v2
-; SI-DENORM-FASTFMA-STRICT-NEXT: v_sub_f32_e32 v2, v3, v2
-; SI-DENORM-FASTFMA-STRICT-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-DENORM-FASTFMA-STRICT-NEXT: s_endpgm
+; SI-DENORM-FASTFMA-LABEL: fsub_c_fadd_a_a_f32:
+; SI-DENORM-FASTFMA: ; %bb.0:
+; SI-DENORM-FASTFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-DENORM-FASTFMA-NEXT: s_mov_b32 s3, 0xf000
+; SI-DENORM-FASTFMA-NEXT: s_mov_b32 s2, 0
+; SI-DENORM-FASTFMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-DENORM-FASTFMA-NEXT: v_mov_b32_e32 v1, 0
+; SI-DENORM-FASTFMA-NEXT: s_waitcnt lgkmcnt(0)
+; SI-DENORM-FASTFMA-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-DENORM-FASTFMA-NEXT: s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMA-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMA-NEXT: s_waitcnt vmcnt(0)
+; SI-DENORM-FASTFMA-NEXT: v_add_f32_e32 v2, v2, v2
+; SI-DENORM-FASTFMA-NEXT: v_sub_f32_e32 v2, v3, v2
+; SI-DENORM-FASTFMA-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMA-NEXT: s_endpgm
;
; SI-DENORM-SLOWFMA-LABEL: fsub_c_fadd_a_a_f32:
; SI-DENORM-SLOWFMA: ; %bb.0:
@@ -2600,22 +2607,6 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f32(ptr addrspace(1) %out, ptr addrsp
; SI-DENORM-SLOWFMA-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMA-NEXT: s_endpgm
;
-; SI-DENORM-FASTFMA-CONTRACT-LABEL: fsub_c_fadd_a_a_f32:
-; SI-DENORM-FASTFMA-CONTRACT: ; %bb.0:
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_mov_b32 s3, 0xf000
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_mov_b32 s2, 0
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: v_fma_f32 v2, v2, -2.0, v3
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-DENORM-FASTFMA-CONTRACT-NEXT: s_endpgm
-;
; GFX9-FLUSH-LABEL: fsub_c_fadd_a_a_f32:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -2625,8 +2616,9 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f32(ptr addrspace(1) %out, ptr addrsp
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLUSH-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc
; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLUSH-NEXT: v_mac_f32_e32 v2, -2.0, v1
-; GFX9-FLUSH-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-FLUSH-NEXT: v_add_f32_e32 v1, v1, v1
+; GFX9-FLUSH-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX9-FLUSH-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-FLUSH-NEXT: s_endpgm
;
; GFX9-DENORM-LABEL: fsub_c_fadd_a_a_f32:
@@ -2643,32 +2635,19 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f32(ptr addrspace(1) %out, ptr addrsp
; GFX9-DENORM-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-DENORM-NEXT: s_endpgm
;
-; GFX10-FLUSH-LABEL: fsub_c_fadd_a_a_f32:
-; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
-; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc dlc
-; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: v_mac_f32_e32 v2, -2.0, v1
-; GFX10-FLUSH-NEXT: global_store_dword v0, v2, s[0:1]
-; GFX10-FLUSH-NEXT: s_endpgm
-;
-; GFX10-DENORM-LABEL: fsub_c_fadd_a_a_f32:
-; GFX10-DENORM: ; %bb.0:
-; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-DENORM-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc dlc
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-NEXT: v_add_f32_e32 v1, v1, v1
-; GFX10-DENORM-NEXT: v_sub_f32_e32 v1, v2, v1
-; GFX10-DENORM-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-DENORM-NEXT: s_endpgm
+; GFX10-LABEL: fsub_c_fadd_a_a_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: global_load_dword v2, v0, s[0:1] offset:4 glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v1
+; GFX10-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep.0 = getelementptr float, ptr addrspace(1) %out, i32 %tid
%gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
index 13e4206ab7f57..4f253c7989a6c 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
@@ -698,22 +698,26 @@ define half @v_fneg_mul_f16(half %a, half %b) #0 {
; SI-LABEL: v_fneg_mul_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, v0, -v1
+; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%mul = fmul half %a, %b
%fneg = fneg half %mul
@@ -756,11 +760,12 @@ define { half, half } @v_fneg_mul_multi_use_mul_f16(half %a, half %b) #0 {
; SI-LABEL: v_fneg_mul_multi_use_mul_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: v_mul_f32_e32 v1, 4.0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -768,16 +773,18 @@ define { half, half } @v_fneg_mul_multi_use_mul_f16(half %a, half %b) #0 {
; VI-LABEL: v_fneg_mul_multi_use_mul_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, v0, -v1
-; VI-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; VI-NEXT: v_mul_f16_e32 v1, v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v1
+; VI-NEXT: v_mul_f16_e32 v1, 4.0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_multi_use_mul_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-NEXT: v_mul_f16_e32 v1, v0, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v1
+; GFX11-NEXT: v_mul_f16_e32 v1, 4.0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%mul = fmul half %a, %b
%fneg = fneg half %mul
@@ -792,21 +799,25 @@ define half @v_fneg_mul_fneg_x_f16(half %a, half %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_fneg_x_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_mul_f16_e64 v0, -v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_fneg_x_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: v_mul_f16_e64 v0, -v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
%mul = fmul half %fneg.a, %b
@@ -818,22 +829,26 @@ define half @v_fneg_mul_x_fneg_f16(half %a, half %b) #0 {
; SI-LABEL: v_fneg_mul_x_fneg_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_x_fneg_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_mul_f16_e64 v0, v0, -v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_x_fneg_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.b = fneg half %b
%mul = fmul half %a, %fneg.b
@@ -845,22 +860,26 @@ define half @v_fneg_mul_fneg_fneg_f16(half %a, half %b) #0 {
; SI-LABEL: v_fneg_mul_fneg_fneg_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_fneg_fneg_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, v0, -v1
+; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_fneg_fneg_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
%fneg.b = fneg half %b
@@ -873,28 +892,31 @@ define { half, half } @v_fneg_mul_store_use_fneg_x_f16(half %a, half %b) #0 {
; SI-LABEL: v_fneg_mul_store_use_fneg_x_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; SI-NEXT: v_mul_f32_e32 v1, v2, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v1
-; SI-NEXT: v_xor_b32_e32 v1, 0xffff8000, v0
-; SI-NEXT: v_mov_b32_e32 v0, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0xffff8000, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_mov_b32_e32 v1, v2
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_store_use_fneg_x_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_xor_b32_e32 v2, 0x8000, v0
-; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_mul_f16_e64 v0, -v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_mul_store_use_fneg_x_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX11-NEXT: v_mul_f16_e64 v1, -v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_xor_b32_e32 v2, 0x8000, v1
; GFX11-NEXT: v_xor_b32_e32 v1, 0x8000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
@@ -910,19 +932,20 @@ define { half, half } @v_fneg_mul_multi_use_fneg_x_f16(half %a, half %b, half %c
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e64 v4, -v0
-; SI-NEXT: v_mul_f32_e32 v0, v3, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, v4, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v1, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v1
+; SI-NEXT: v_mul_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_mul_multi_use_fneg_x_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e32 v3, v0, v1
+; VI-NEXT: v_mul_f16_e64 v1, -v0, v1
+; VI-NEXT: v_xor_b32_e32 v3, 0x8000, v1
; VI-NEXT: v_mul_f16_e64 v1, -v0, v2
; VI-NEXT: v_mov_b32_e32 v0, v3
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -930,9 +953,10 @@ define { half, half } @v_fneg_mul_multi_use_fneg_x_f16(half %a, half %b, half %c
; GFX11-LABEL: v_fneg_mul_multi_use_fneg_x_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e32 v3, v0, v1
+; GFX11-NEXT: v_mul_f16_e64 v1, -v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_xor_b32_e32 v3, 0x8000, v1
; GFX11-NEXT: v_mul_f16_e64 v1, -v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
@@ -2229,8 +2253,8 @@ define { half, half } @v_fneg_fma_multi_use_fma_f16_nsz(half %a, half %b, half %
; SI-LABEL: v_fneg_fma_multi_use_fma_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; SI-NEXT: v_cvt_f32_f16_e64 v3, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
@@ -2279,7 +2303,8 @@ define { half, half } @v_fneg_fma_multi_use_fma_f16_nsz(half %a, half %b, half %
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: v_mul_f32_e32 v1, 4.0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -2287,16 +2312,18 @@ define { half, half } @v_fneg_fma_multi_use_fma_f16_nsz(half %a, half %b, half %
; VI-LABEL: v_fneg_fma_multi_use_fma_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, -v1, -v2
-; VI-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; VI-NEXT: v_fma_f16 v1, v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v1
+; VI-NEXT: v_mul_f16_e32 v1, 4.0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_multi_use_fma_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; GFX11-NEXT: v_fmac_f16_e32 v2, v0, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, 4.0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz half @llvm.fma.f16(half %a, half %b, half %c)
%fneg = fneg half %fma
@@ -2387,9 +2414,9 @@ define half @v_fneg_fma_fneg_x_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fma_fneg_x_y_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v4, -v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
@@ -2437,18 +2464,22 @@ define half @v_fneg_fma_fneg_x_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_fneg_x_y_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, v1, -v2
+; VI-NEXT: v_fma_f16 v0, -v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_fneg_x_y_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX11-NEXT: v_fma_f16 v0, -v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
%fma = call nsz half @llvm.fma.f16(half %fneg.a, half %b, half %c)
@@ -2537,8 +2568,8 @@ define half @v_fneg_fma_x_fneg_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fma_x_fneg_y_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e64 v3, -v1
; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
@@ -2587,18 +2618,22 @@ define half @v_fneg_fma_x_fneg_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_x_fneg_y_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, v1, -v2
+; VI-NEXT: v_fma_f16 v0, v0, -v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_x_fneg_y_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX11-NEXT: v_fma_f16 v0, v0, -v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.b = fneg half %b
%fma = call nsz half @llvm.fma.f16(half %a, half %fneg.b, half %c)
@@ -2688,8 +2723,8 @@ define half @v_fneg_fma_fneg_fneg_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fma_fneg_fneg_y_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; SI-NEXT: v_cvt_f32_f16_e64 v3, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
@@ -2738,18 +2773,22 @@ define half @v_fneg_fma_fneg_fneg_y_f16_nsz(half %a, half %b, half %c) #0 {
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_fneg_fneg_y_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; VI-NEXT: v_fma_f16 v0, v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_fneg_fneg_y_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; GFX11-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
%fneg.b = fneg half %b
@@ -2840,9 +2879,9 @@ define half @v_fneg_fma_fneg_x_fneg_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fma_fneg_x_fneg_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v4, -v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
@@ -2890,18 +2929,22 @@ define half @v_fneg_fma_fneg_x_fneg_f16_nsz(half %a, half %b, half %c) #0 {
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_fneg_x_fneg_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, v1, v2
+; VI-NEXT: v_fma_f16 v0, -v0, v1, -v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_fneg_x_fneg_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX11-NEXT: v_fma_f16 v0, -v0, v1, -v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
%fneg.c = fneg half %c
@@ -2991,8 +3034,8 @@ define half @v_fneg_fma_x_y_fneg_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fma_x_y_fneg_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e64 v3, -v1
+; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v4, v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
@@ -3041,18 +3084,22 @@ define half @v_fneg_fma_x_y_fneg_f16_nsz(half %a, half %b, half %c) #0 {
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_x_y_fneg_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, -v1, v2
+; VI-NEXT: v_fma_f16 v0, v0, v1, -v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_x_y_fneg_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, -v1, v2
+; GFX11-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.c = fneg half %c
%fma = call nsz half @llvm.fma.f16(half %a, half %b, half %fneg.c)
@@ -3149,22 +3196,22 @@ define { half, half } @v_fneg_fma_store_use_fneg_x_y_f16_nsz(half %a, half %b, h
; SI-LABEL: v_fneg_fma_store_use_fneg_x_y_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v0
+; SI-NEXT: v_xor_b32_e32 v3, 0xffff8000, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v3
; SI-NEXT: s_movk_i32 s4, 0x3f1
-; SI-NEXT: v_cvt_f64_f32_e32 v[1:2], v2
-; SI-NEXT: v_cvt_f64_f32_e32 v[3:4], v3
-; SI-NEXT: v_cvt_f64_f32_e32 v[5:6], v5
-; SI-NEXT: v_fma_f64 v[2:3], v[5:6], v[3:4], v[1:2]
-; SI-NEXT: v_xor_b32_e32 v1, 0xffff8000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0x1ff, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v2
+; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
+; SI-NEXT: v_fma_f64 v[0:1], v[6:7], v[4:5], v[0:1]
+; SI-NEXT: v_and_b32_e32 v2, 0x1ff, v1
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v3
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v1
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SI-NEXT: v_and_b32_e32 v2, 0xffe, v2
-; SI-NEXT: v_bfe_u32 v4, v3, 20, 11
+; SI-NEXT: v_bfe_u32 v4, v1, 20, 11
; SI-NEXT: v_or_b32_e32 v0, v2, v0
; SI-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
; SI-NEXT: v_or_b32_e32 v2, 0x1000, v0
@@ -3196,26 +3243,30 @@ define { half, half } @v_fneg_fma_store_use_fneg_x_y_f16_nsz(half %a, half %b, h
; SI-NEXT: s_movk_i32 s4, 0x40f
; SI-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
; SI-NEXT: v_cmp_eq_u32_e32 vcc, s4, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; SI-NEXT: v_and_b32_e32 v2, 0x8000, v2
-; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
+; SI-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
+; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_store_use_fneg_x_y_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_xor_b32_e32 v3, 0x8000, v0
-; VI-NEXT: v_fma_f16 v0, v0, v1, -v2
+; VI-NEXT: v_fma_f16 v0, -v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fma_store_use_fneg_x_y_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v2, v0, v1, -v2
+; GFX11-NEXT: v_fma_f16 v1, -v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_xor_b32_e32 v2, 0x8000, v1
; GFX11-NEXT: v_xor_b32_e32 v1, 0x8000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
@@ -3317,66 +3368,67 @@ define { half, half } @v_fneg_fma_multi_use_fneg_x_y_f16_nsz(half %a, half %b, h
; SI-LABEL: v_fneg_fma_multi_use_fneg_x_y_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v2
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v8, -v0
; SI-NEXT: s_movk_i32 s4, 0x3f1
-; SI-NEXT: v_cvt_f64_f32_e32 v[1:2], v2
+; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
+; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v8
; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; SI-NEXT: v_fma_f64 v[1:2], v[6:7], v[4:5], v[1:2]
-; SI-NEXT: v_and_b32_e32 v4, 0x1ff, v2
-; SI-NEXT: v_or_b32_e32 v1, v4, v1
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v4, 8, v2
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; SI-NEXT: v_and_b32_e32 v4, 0xffe, v4
-; SI-NEXT: v_bfe_u32 v5, v2, 20, 11
-; SI-NEXT: v_or_b32_e32 v1, v4, v1
-; SI-NEXT: v_sub_i32_e32 v6, vcc, s4, v5
-; SI-NEXT: v_or_b32_e32 v4, 0x1000, v1
-; SI-NEXT: v_med3_i32 v6, v6, 0, 13
-; SI-NEXT: v_lshrrev_b32_e32 v7, v6, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, v6, v7
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, v6, v4
+; SI-NEXT: v_fma_f64 v[0:1], v[6:7], v[4:5], v[0:1]
+; SI-NEXT: v_and_b32_e32 v2, 0x1ff, v1
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: v_and_b32_e32 v2, 0xffe, v2
+; SI-NEXT: v_bfe_u32 v4, v1, 20, 11
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
+; SI-NEXT: v_or_b32_e32 v2, 0x1000, v0
+; SI-NEXT: v_med3_i32 v5, v5, 0, 13
+; SI-NEXT: v_lshrrev_b32_e32 v6, v5, v2
+; SI-NEXT: v_lshlrev_b32_e32 v5, v5, v6
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, v5, v2
; SI-NEXT: s_movk_i32 s4, 0xfc10
-; SI-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; SI-NEXT: v_add_i32_e32 v5, vcc, s4, v5
-; SI-NEXT: v_lshlrev_b32_e32 v6, 12, v5
-; SI-NEXT: v_or_b32_e32 v4, v7, v4
-; SI-NEXT: v_or_b32_e32 v6, v1, v6
-; SI-NEXT: v_cmp_gt_i32_e32 vcc, 1, v5
-; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_and_b32_e32 v6, 7, v4
-; SI-NEXT: v_cmp_lt_i32_e32 vcc, 5, v6
-; SI-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, 3, v6
+; SI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; SI-NEXT: v_add_i32_e32 v4, vcc, s4, v4
+; SI-NEXT: v_lshlrev_b32_e32 v5, 12, v4
+; SI-NEXT: v_or_b32_e32 v2, v6, v2
+; SI-NEXT: v_or_b32_e32 v5, v0, v5
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
+; SI-NEXT: v_and_b32_e32 v5, 7, v2
+; SI-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
; SI-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; SI-NEXT: v_or_b32_e32 v6, v6, v7
-; SI-NEXT: v_lshrrev_b32_e32 v4, 2, v4
-; SI-NEXT: v_add_i32_e32 v4, vcc, v4, v6
-; SI-NEXT: v_mov_b32_e32 v6, 0x7c00
-; SI-NEXT: v_cmp_gt_i32_e32 vcc, 31, v5
-; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v7, 0x7e00
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
+; SI-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_lshrrev_b32_e32 v2, 2, v2
+; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v5
+; SI-NEXT: v_mov_b32_e32 v5, 0x7c00
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
+; SI-NEXT: v_mov_b32_e32 v6, 0x7e00
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; SI-NEXT: s_movk_i32 s4, 0x40f
-; SI-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, s4, v5
-; SI-NEXT: v_mul_f32_e32 v0, v0, v3
-; SI-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v0, 0x8000, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v4
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, s4, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-NEXT: v_and_b32_e32 v2, 0x8000, v1
+; SI-NEXT: v_mul_f32_e32 v1, v8, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fma_multi_use_fneg_x_y_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v2, v0, v1, -v2
+; VI-NEXT: v_fma_f16 v1, -v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x8000, v1
; VI-NEXT: v_mul_f16_e64 v1, -v0, v3
; VI-NEXT: v_mov_b32_e32 v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3384,9 +3436,10 @@ define { half, half } @v_fneg_fma_multi_use_fneg_x_y_f16_nsz(half %a, half %b, h
; GFX11-LABEL: v_fneg_fma_multi_use_fneg_x_y_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v2, v0, v1, -v2
+; GFX11-NEXT: v_fma_f16 v1, -v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_xor_b32_e32 v2, 0x8000, v1
; GFX11-NEXT: v_mul_f16_e64 v1, -v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg half %a
@@ -3440,12 +3493,12 @@ define half @v_fneg_fmad_f16_nsz(half %a, half %b, half %c) #0 {
; SI-LABEL: v_fneg_fmad_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; SI-NEXT: v_sub_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -3453,13 +3506,16 @@ define half @v_fneg_fmad_f16_nsz(half %a, half %b, half %c) #0 {
; VI-LABEL: v_fneg_fmad_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; VI-NEXT: v_fma_f16 v0, v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fmad_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; GFX11-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz half @llvm.fmuladd.f16(half %a, half %b, half %c)
%fneg = fneg half %fma
@@ -3555,50 +3611,56 @@ define <4 x half> @v_fneg_fmad_v4f32_nsz(<4 x half> %a, <4 x half> %b, <4 x half
; SI-LABEL: v_fneg_fmad_v4f32_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
; SI-NEXT: v_mul_f32_e32 v6, v6, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v10
; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v7, v9, v7
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v7, v7, v9
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_mul_f32_e32 v1, v1, v3
-; SI-NEXT: v_mul_f32_e32 v0, v0, v2
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v0, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v8
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_or_b32_e32 v0, v0, v6
+; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; SI-NEXT: v_sub_f32_e32 v2, v2, v3
; SI-NEXT: v_cvt_f32_f16_e32 v3, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v7
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_sub_f32_e32 v6, v6, v8
-; SI-NEXT: v_sub_f32_e32 v2, v7, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_sub_f32_e32 v1, v1, v3
+; SI-NEXT: v_sub_f32_e32 v3, v5, v6
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_sub_f32_e32 v0, v0, v4
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_fneg_fmad_v4f32_nsz:
@@ -3607,24 +3669,29 @@ define <4 x half> @v_fneg_fmad_v4f32_nsz(<4 x half> %a, <4 x half> %b, <4 x half
; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; VI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; VI-NEXT: v_fma_f16 v6, v8, -v7, -v6
+; VI-NEXT: v_fma_f16 v6, v8, v7, v6
; VI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; VI-NEXT: v_fma_f16 v7, v9, -v8, -v7
-; VI-NEXT: v_fma_f16 v0, v0, -v2, -v4
+; VI-NEXT: v_fma_f16 v7, v9, v8, v7
+; VI-NEXT: v_fma_f16 v0, v0, v2, v4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v7
-; VI-NEXT: v_fma_f16 v1, v1, -v3, -v5
+; VI-NEXT: v_fma_f16 v1, v1, v3, v5
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; VI-NEXT: v_or_b32_e32 v1, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fmad_v4f32_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_fma_f16 v0, v0, v2, v4 neg_lo:[0,1,1] neg_hi:[0,1,1]
-; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v5 neg_lo:[0,1,1] neg_hi:[0,1,1]
+; GFX11-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX11-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
%fneg = fneg <4 x half> %fma
@@ -3677,12 +3744,12 @@ define { half, half } @v_fneg_fmad_multi_use_fmad_f16_nsz(half %a, half %b, half
; SI-LABEL: v_fneg_fmad_multi_use_fmad_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; SI-NEXT: v_sub_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_cvt_f32_f16_e64 v1, -v0
@@ -3693,16 +3760,18 @@ define { half, half } @v_fneg_fmad_multi_use_fmad_f16_nsz(half %a, half %b, half
; VI-LABEL: v_fneg_fmad_multi_use_fmad_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v0, v0, -v1, -v2
-; VI-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; VI-NEXT: v_fma_f16 v1, v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v1
+; VI-NEXT: v_mul_f16_e32 v1, 4.0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fneg_fmad_multi_use_fmad_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; GFX11-NEXT: v_fmac_f16_e32 v2, v0, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v1, -4.0, v0
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v2
+; GFX11-NEXT: v_mul_f16_e32 v1, 4.0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz half @llvm.fmuladd.f16(half %a, half %b, half %c)
%fneg = fneg half %fma
@@ -5044,10 +5113,11 @@ define half @v_fneg_inlineasm_f16(half %a, half %b, half %c, i32 %d) #0 {
; SI-LABEL: v_fneg_inlineasm_f16:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ; use v0
; SI-NEXT: ;;#ASMEND
@@ -5056,7 +5126,8 @@ define half @v_fneg_inlineasm_f16(half %a, half %b, half %c, i32 %d) #0 {
; VI-LABEL: v_fneg_inlineasm_f16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, v0, -v1
+; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; VI-NEXT: ;;#ASMSTART
; VI-NEXT: ; use v0
; VI-NEXT: ;;#ASMEND
@@ -5065,7 +5136,9 @@ define half @v_fneg_inlineasm_f16(half %a, half %b, half %c, i32 %d) #0 {
; GFX11-LABEL: v_fneg_inlineasm_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: ; use v0
; GFX11-NEXT: ;;#ASMEND
@@ -5469,12 +5542,12 @@ define { half, half } @free_fold_src_code_size_cost_use_f16_nsz(ptr addrspace(1)
; SI-LABEL: free_fold_src_code_size_cost_use_f16_nsz:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v0, -v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v3
; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
; SI-NEXT: s_movk_i32 s4, 0x3f1
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], -2.0
+; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], 2.0
; SI-NEXT: v_and_b32_e32 v2, 0x1ff, v1
; SI-NEXT: v_or_b32_e32 v0, v2, v0
; SI-NEXT: v_lshrrev_b32_e32 v3, 8, v1
@@ -5517,7 +5590,7 @@ define { half, half } @free_fold_src_code_size_cost_use_f16_nsz(ptr addrspace(1)
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; SI-NEXT: v_and_b32_e32 v1, 0x8000, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v1, -v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, v4
; SI-NEXT: v_cvt_f32_f16_e32 v2, v5
; SI-NEXT: v_mul_f32_e32 v0, v1, v0
@@ -5529,18 +5602,18 @@ define { half, half } @free_fold_src_code_size_cost_use_f16_nsz(ptr addrspace(1)
; VI-LABEL: free_fold_src_code_size_cost_use_f16_nsz:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_fma_f16 v1, v2, -v3, -2.0
-; VI-NEXT: v_mul_f16_e32 v0, v1, v4
-; VI-NEXT: v_mul_f16_e32 v1, v1, v5
+; VI-NEXT: v_fma_f16 v1, v2, v3, 2.0
+; VI-NEXT: v_mul_f16_e64 v0, -v1, v4
+; VI-NEXT: v_mul_f16_e64 v1, -v1, v5
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: free_fold_src_code_size_cost_use_f16_nsz:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f16 v1, v2, -v3, -2.0
+; GFX11-NEXT: v_fma_f16 v1, v2, v3, 2.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v0, v1, v4
-; GFX11-NEXT: v_mul_f16_e32 v1, v1, v5
+; GFX11-NEXT: v_mul_f16_e64 v0, -v1, v4
+; GFX11-NEXT: v_mul_f16_e64 v1, -v1, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -5946,12 +6019,11 @@ define half @denorm_snan_fmul_neg1_to_fneg(half %x, half %y) {
; SI-LABEL: denorm_snan_fmul_neg1_to_fneg:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v2, -v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, v0, v2
+; SI-NEXT: v_mul_f32_e32 v0, v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; SI-NEXT: v_mul_f32_e32 v0, v0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -5959,16 +6031,16 @@ define half @denorm_snan_fmul_neg1_to_fneg(half %x, half %y) {
; VI-LABEL: denorm_snan_fmul_neg1_to_fneg:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, v0, -v0
-; VI-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-NEXT: v_mul_f16_e32 v0, v0, v0
+; VI-NEXT: v_mul_f16_e64 v0, -v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: denorm_snan_fmul_neg1_to_fneg:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v0
+; GFX11-NEXT: v_mul_f16_e32 v0, v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX11-NEXT: v_mul_f16_e64 v0, -v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%canonical = fmul half %x, %x
%mul = fmul half %canonical, -1.0
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
index e50489cdeba54..7533d67950b7a 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
@@ -85,11 +85,11 @@ define { float, float } @v_fneg_interp_p1_f16(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_interp_p1_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v1, v0, -v1
+; GCN-NEXT: v_mul_f32_e32 v1, v0, v1
; GCN-NEXT: s_mov_b32 m0, 0
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
-; GCN-NEXT: v_interp_p1ll_f16 v0, v1, attr0.x
-; GCN-NEXT: v_interp_p1ll_f16 v1, v1, attr0.y
+; GCN-NEXT: v_interp_p1ll_f16 v0, -v1, attr0.x
+; GCN-NEXT: v_interp_p1ll_f16 v1, -v1, attr0.y
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
@@ -104,12 +104,12 @@ define { half, half } @v_fneg_interp_p2_f16(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_interp_p2_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v1, v0, -v1
+; GCN-NEXT: v_mul_f32_e32 v1, v0, v1
; GCN-NEXT: v_mov_b32_e32 v2, 4.0
; GCN-NEXT: s_mov_b32 m0, 0
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
-; GCN-NEXT: v_interp_p2_f16 v0, v1, attr0.x, v2
-; GCN-NEXT: v_interp_p2_f16 v1, v1, attr0.y, v2
+; GCN-NEXT: v_interp_p2_f16 v0, -v1, attr0.x, v2
+; GCN-NEXT: v_interp_p2_f16 v1, -v1, attr0.y, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
index cfa5247267559..64b48eb6ee473 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -1090,7 +1090,8 @@ define amdgpu_kernel void @v_fneg_mul_f32(ptr addrspace(1) %out, ptr addrspace(1
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v4
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; SI-NEXT: v_mul_f32_e32 v2, v5, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -1113,7 +1114,8 @@ define amdgpu_kernel void @v_fneg_mul_f32(ptr addrspace(1) %out, ptr addrspace(1
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; VI-NEXT: v_mul_f32_e32 v2, v5, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1214,12 +1216,13 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_mul_f32(ptr addrspace(1) %out, p
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mul_f32_e64 v2, v4, -v2
-; SI-NEXT: v_mul_f32_e32 v3, -4.0, v2
-; SI-NEXT: flat_store_dword v[0:1], v2
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mul_f32_e32 v2, v4, v2
+; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; SI-NEXT: v_mul_f32_e32 v2, 4.0, v2
; SI-NEXT: flat_store_dword v[0:1], v3
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: flat_store_dword v[0:1], v2
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_fneg_mul_multi_use_mul_f32:
@@ -1240,12 +1243,13 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_mul_f32(ptr addrspace(1) %out, p
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mul_f32_e64 v2, v4, -v2
-; VI-NEXT: v_mul_f32_e32 v3, -4.0, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mul_f32_e32 v2, v4, v2
+; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; VI-NEXT: v_mul_f32_e32 v2, 4.0, v2
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -1279,7 +1283,8 @@ define amdgpu_kernel void @v_fneg_mul_fneg_x_f32(ptr addrspace(1) %out, ptr addr
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v2, v0, v1
+; SI-NEXT: v_mul_f32_e64 v0, -v0, v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -1302,7 +1307,8 @@ define amdgpu_kernel void @v_fneg_mul_fneg_x_f32(ptr addrspace(1) %out, ptr addr
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_f32_e32 v2, v0, v1
+; VI-NEXT: v_mul_f32_e64 v0, -v0, v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -1339,7 +1345,8 @@ define amdgpu_kernel void @v_fneg_mul_x_fneg_f32(ptr addrspace(1) %out, ptr addr
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v2, v0, v1
+; SI-NEXT: v_mul_f32_e64 v0, v0, -v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -1362,7 +1369,8 @@ define amdgpu_kernel void @v_fneg_mul_x_fneg_f32(ptr addrspace(1) %out, ptr addr
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_f32_e32 v2, v0, v1
+; VI-NEXT: v_mul_f32_e64 v0, v0, -v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -1399,7 +1407,8 @@ define amdgpu_kernel void @v_fneg_mul_fneg_fneg_f32(ptr addrspace(1) %out, ptr a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, v0, -v1
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -1422,7 +1431,8 @@ define amdgpu_kernel void @v_fneg_mul_fneg_fneg_f32(ptr addrspace(1) %out, ptr a
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v2, v0, -v1
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -1463,7 +1473,8 @@ define amdgpu_kernel void @v_fneg_mul_store_use_fneg_x_f32(ptr addrspace(1) %out
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v4
-; SI-NEXT: v_mul_f32_e32 v2, v4, v2
+; SI-NEXT: v_mul_f32_e64 v2, -v4, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v3
@@ -1489,7 +1500,8 @@ define amdgpu_kernel void @v_fneg_mul_store_use_fneg_x_f32(ptr addrspace(1) %out
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v4
-; VI-NEXT: v_mul_f32_e32 v2, v4, v2
+; VI-NEXT: v_mul_f32_e64 v2, -v4, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3
@@ -1532,7 +1544,8 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_fneg_x_f32(ptr addrspace(1) %out
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mul_f32_e64 v3, -v4, s2
-; SI-NEXT: v_mul_f32_e32 v2, v4, v2
+; SI-NEXT: v_mul_f32_e64 v2, -v4, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v3
@@ -1560,7 +1573,8 @@ define amdgpu_kernel void @v_fneg_mul_multi_use_fneg_x_f32(ptr addrspace(1) %out
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mul_f32_e64 v3, -v4, s2
-; VI-NEXT: v_mul_f32_e32 v2, v4, v2
+; VI-NEXT: v_mul_f32_e64 v2, -v4, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3
@@ -3062,7 +3076,8 @@ define amdgpu_kernel void @v_fneg_fma_f32_nsz(ptr addrspace(1) %out, ptr addrspa
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v6
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_fma_f32 v2, v7, -v2, -v3
+; SI-NEXT: v_fma_f32 v2, v7, v2, v3
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -3089,7 +3104,8 @@ define amdgpu_kernel void @v_fneg_fma_f32_nsz(ptr addrspace(1) %out, ptr addrspa
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v6
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_fma_f32 v2, v7, -v2, -v3
+; VI-NEXT: v_fma_f32 v2, v7, v2, v3
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -3361,12 +3377,13 @@ define amdgpu_kernel void @v_fneg_fma_multi_use_fma_f32_nsz(ptr addrspace(1) %ou
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_fma_f32 v2, v6, -v2, -v3
-; SI-NEXT: v_mul_f32_e32 v3, -4.0, v2
-; SI-NEXT: flat_store_dword v[0:1], v2
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_fma_f32 v2, v6, v2, v3
+; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; SI-NEXT: v_mul_f32_e32 v2, 4.0, v2
; SI-NEXT: flat_store_dword v[0:1], v3
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: flat_store_dword v[0:1], v2
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_fneg_fma_multi_use_fma_f32_nsz:
@@ -3391,12 +3408,13 @@ define amdgpu_kernel void @v_fneg_fma_multi_use_fma_f32_nsz(ptr addrspace(1) %ou
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_fma_f32 v2, v6, -v2, -v3
-; VI-NEXT: v_mul_f32_e32 v3, -4.0, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_fma_f32 v2, v6, v2, v3
+; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; VI-NEXT: v_mul_f32_e32 v2, 4.0, v2
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -3508,7 +3526,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_y_f32_nsz(ptr addrspace(1) %out, pt
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[4:5] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_fma_f32 v2, v0, v1, -v2
+; SI-NEXT: v_fma_f32 v0, -v0, v1, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -3535,7 +3554,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_y_f32_nsz(ptr addrspace(1) %out, pt
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_fma_f32 v2, v0, v1, -v2
+; VI-NEXT: v_fma_f32 v0, -v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -3650,7 +3670,8 @@ define amdgpu_kernel void @v_fneg_fma_x_fneg_y_f32_nsz(ptr addrspace(1) %out, pt
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_fma_f32 v2, v0, v1, -v2
+; SI-NEXT: v_fma_f32 v0, v0, -v1, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -3677,7 +3698,8 @@ define amdgpu_kernel void @v_fneg_fma_x_fneg_y_f32_nsz(ptr addrspace(1) %out, pt
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_fma_f32 v2, v0, v1, -v2
+; VI-NEXT: v_fma_f32 v0, v0, -v1, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -3793,7 +3815,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_fneg_y_f32_nsz(ptr addrspace(1) %out,
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[4:5] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_fma_f32 v2, v0, -v1, -v2
+; SI-NEXT: v_fma_f32 v0, v0, v1, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -3820,7 +3843,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_fneg_y_f32_nsz(ptr addrspace(1) %out,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_fma_f32 v2, v0, -v1, -v2
+; VI-NEXT: v_fma_f32 v0, v0, v1, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -3937,7 +3961,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_fneg_f32_nsz(ptr addrspace(1) %out,
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[4:5] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_fma_f32 v2, v0, v1, v2
+; SI-NEXT: v_fma_f32 v0, -v0, v1, -v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -3964,7 +3989,8 @@ define amdgpu_kernel void @v_fneg_fma_fneg_x_fneg_f32_nsz(ptr addrspace(1) %out,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_fma_f32 v2, v0, v1, v2
+; VI-NEXT: v_fma_f32 v0, -v0, v1, -v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -4080,7 +4106,8 @@ define amdgpu_kernel void @v_fneg_fma_x_y_fneg_f32_nsz(ptr addrspace(1) %out, pt
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[4:5] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_fma_f32 v2, v0, -v1, v2
+; SI-NEXT: v_fma_f32 v0, v0, v1, -v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -4107,7 +4134,8 @@ define amdgpu_kernel void @v_fneg_fma_x_y_fneg_f32_nsz(ptr addrspace(1) %out, pt
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_fma_f32 v2, v0, -v1, v2
+; VI-NEXT: v_fma_f32 v0, v0, v1, -v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -4232,7 +4260,8 @@ define amdgpu_kernel void @v_fneg_fma_store_use_fneg_x_y_f32_nsz(ptr addrspace(1
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_xor_b32_e32 v4, 0x80000000, v6
-; SI-NEXT: v_fma_f32 v2, v6, v2, -v3
+; SI-NEXT: v_fma_f32 v2, -v6, v2, v3
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v4
@@ -4262,7 +4291,8 @@ define amdgpu_kernel void @v_fneg_fma_store_use_fneg_x_y_f32_nsz(ptr addrspace(1
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_xor_b32_e32 v4, 0x80000000, v6
-; VI-NEXT: v_fma_f32 v2, v6, v2, -v3
+; VI-NEXT: v_fma_f32 v2, -v6, v2, v3
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v4
@@ -4466,7 +4496,7 @@ define amdgpu_kernel void @v_fneg_fmad_f32_nsz(ptr addrspace(1) %out, ptr addrsp
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v6
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mad_f32 v2, v7, -v2, -v3
+; SI-NEXT: v_mad_f32 v2, -v7, v2, -v3
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -4493,7 +4523,7 @@ define amdgpu_kernel void @v_fneg_fmad_f32_nsz(ptr addrspace(1) %out, ptr addrsp
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v6
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mad_f32 v2, v7, -v2, -v3
+; VI-NEXT: v_mad_f32 v2, -v7, v2, -v3
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -4618,10 +4648,10 @@ define amdgpu_kernel void @v_fneg_fmad_v4f32_nsz(ptr addrspace(1) %out, ptr addr
; SI-NEXT: v_mov_b32_e32 v13, s1
; SI-NEXT: v_add_i32_e32 v12, vcc, s0, v12
; SI-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; SI-NEXT: v_mad_f32 v3, v3, -v7, -v11
-; SI-NEXT: v_mad_f32 v2, v2, -v6, -v10
-; SI-NEXT: v_mad_f32 v1, v1, -v5, -v9
-; SI-NEXT: v_mad_f32 v0, v0, -v4, -v8
+; SI-NEXT: v_mad_f32 v3, -v3, v7, -v11
+; SI-NEXT: v_mad_f32 v2, -v2, v6, -v10
+; SI-NEXT: v_mad_f32 v1, -v1, v5, -v9
+; SI-NEXT: v_mad_f32 v0, -v0, v4, -v8
; SI-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
; SI-NEXT: s_endpgm
;
@@ -4648,10 +4678,10 @@ define amdgpu_kernel void @v_fneg_fmad_v4f32_nsz(ptr addrspace(1) %out, ptr addr
; VI-NEXT: v_mov_b32_e32 v13, s1
; VI-NEXT: v_add_u32_e32 v12, vcc, s0, v12
; VI-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
-; VI-NEXT: v_mad_f32 v3, v3, -v7, -v11
-; VI-NEXT: v_mad_f32 v2, v2, -v6, -v10
-; VI-NEXT: v_mad_f32 v1, v1, -v5, -v9
-; VI-NEXT: v_mad_f32 v0, v0, -v4, -v8
+; VI-NEXT: v_mad_f32 v3, -v3, v7, -v11
+; VI-NEXT: v_mad_f32 v2, -v2, v6, -v10
+; VI-NEXT: v_mad_f32 v1, -v1, v5, -v9
+; VI-NEXT: v_mad_f32 v0, -v0, v4, -v8
; VI-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -5933,7 +5963,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_f32(ptr addrspace(1) %out, ptr addr
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v4
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mul_legacy_f32_e64 v2, v5, -v2
+; SI-NEXT: v_mul_legacy_f32_e32 v2, v5, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -5956,7 +5987,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_f32(ptr addrspace(1) %out, ptr addr
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mul_legacy_f32_e64 v2, v5, -v2
+; VI-NEXT: v_mul_legacy_f32_e32 v2, v5, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -6057,12 +6089,13 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_mul_legacy_f32(ptr addrsp
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mul_legacy_f32_e64 v2, v4, -v2
-; SI-NEXT: v_mul_legacy_f32_e64 v3, -v2, 4.0
-; SI-NEXT: flat_store_dword v[0:1], v2
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; SI-NEXT: v_mul_legacy_f32_e32 v2, 4.0, v2
; SI-NEXT: flat_store_dword v[0:1], v3
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: flat_store_dword v[0:1], v2
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_fneg_mul_legacy_multi_use_mul_legacy_f32:
@@ -6083,12 +6116,13 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_mul_legacy_f32(ptr addrsp
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mul_legacy_f32_e64 v2, v4, -v2
-; VI-NEXT: v_mul_legacy_f32_e64 v3, -v2, 4.0
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v2
+; VI-NEXT: v_mul_legacy_f32_e32 v2, 4.0, v2
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -6122,7 +6156,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_x_f32(ptr addrspace(1) %out, p
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_legacy_f32_e32 v2, v0, v1
+; SI-NEXT: v_mul_legacy_f32_e64 v0, -v0, v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -6145,7 +6180,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_x_f32(ptr addrspace(1) %out, p
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_legacy_f32_e32 v2, v0, v1
+; VI-NEXT: v_mul_legacy_f32_e64 v0, -v0, v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -6182,7 +6218,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_x_fneg_f32(ptr addrspace(1) %out, p
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_legacy_f32_e32 v2, v0, v1
+; SI-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -6205,7 +6242,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_x_fneg_f32(ptr addrspace(1) %out, p
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_legacy_f32_e32 v2, v0, v1
+; VI-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -6242,7 +6280,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_fneg_f32(ptr addrspace(1) %out
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v1, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_legacy_f32_e64 v2, v0, -v1
+; SI-NEXT: v_mul_legacy_f32_e64 v0, -v0, -v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -6265,7 +6304,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_fneg_fneg_f32(ptr addrspace(1) %out
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v1, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mul_legacy_f32_e64 v2, v0, -v1
+; VI-NEXT: v_mul_legacy_f32_e64 v0, -v0, -v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -6306,7 +6346,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_store_use_fneg_x_f32(ptr addrspace(
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v4
-; SI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; SI-NEXT: v_mul_legacy_f32_e64 v2, -v4, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v3
@@ -6332,7 +6373,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_store_use_fneg_x_f32(ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v4
-; VI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; VI-NEXT: v_mul_legacy_f32_e64 v2, -v4, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3
@@ -6375,7 +6417,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_fneg_x_f32(ptr addrspace(
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mul_legacy_f32_e64 v3, -v4, s2
-; SI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; SI-NEXT: v_mul_legacy_f32_e64 v2, -v4, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v3
@@ -6403,7 +6446,8 @@ define amdgpu_kernel void @v_fneg_mul_legacy_multi_use_fneg_x_f32(ptr addrspace(
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mul_legacy_f32_e64 v3, -v4, s2
-; VI-NEXT: v_mul_legacy_f32_e32 v2, v4, v2
+; VI-NEXT: v_mul_legacy_f32_e64 v2, -v4, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3
@@ -6864,7 +6908,8 @@ define amdgpu_kernel void @v_fneg_interp_p1_f32(ptr addrspace(1) %out, ptr addrs
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v4
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; SI-NEXT: v_mul_f32_e32 v2, v5, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: v_interp_p1_f32 v3, v2, attr0.x
; SI-NEXT: v_interp_p1_f32 v2, v2, attr0.y
; SI-NEXT: flat_store_dword v[0:1], v3
@@ -6893,7 +6938,8 @@ define amdgpu_kernel void @v_fneg_interp_p1_f32(ptr addrspace(1) %out, ptr addrs
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; VI-NEXT: v_mul_f32_e32 v2, v5, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: v_interp_p1_f32_e32 v3, v2, attr0.x
; VI-NEXT: v_interp_p1_f32_e32 v2, v2, attr0.y
; VI-NEXT: flat_store_dword v[0:1], v3
@@ -6940,7 +6986,8 @@ define amdgpu_kernel void @v_fneg_interp_p2_f32(ptr addrspace(1) %out, ptr addrs
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v4
; SI-NEXT: v_mov_b32_e32 v3, 4.0
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; SI-NEXT: v_mul_f32_e32 v2, v5, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: v_interp_p2_f32 v6, v2, attr0.x
; SI-NEXT: v_interp_p2_f32 v3, v2, attr0.y
; SI-NEXT: flat_store_dword v[0:1], v6
@@ -6971,7 +7018,8 @@ define amdgpu_kernel void @v_fneg_interp_p2_f32(ptr addrspace(1) %out, ptr addrs
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
; VI-NEXT: v_mov_b32_e32 v3, 4.0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mul_f32_e64 v2, v5, -v2
+; VI-NEXT: v_mul_f32_e32 v2, v5, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: v_interp_p2_f32_e32 v6, v2, attr0.x
; VI-NEXT: v_interp_p2_f32_e32 v3, v2, attr0.y
; VI-NEXT: flat_store_dword v[0:1], v6
@@ -7120,11 +7168,11 @@ define amdgpu_kernel void @v_fneg_inlineasm_f32(ptr addrspace(1) %out, ptr addrs
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: v_add_i32_e32 v2, vcc, s4, v4
; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v5, s7
; SI-NEXT: flat_load_dword v6, v[0:1] glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_load_dword v2, v[2:3] glc
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mov_b32_e32 v5, s7
; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v4
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; SI-NEXT: flat_load_dword v0, v[0:1] glc
@@ -7132,7 +7180,8 @@ define amdgpu_kernel void @v_fneg_inlineasm_f32(ptr addrspace(1) %out, ptr addrs
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v4
; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT: v_mul_f32_e64 v2, v6, -v2
+; SI-NEXT: v_mul_f32_e32 v2, v6, v2
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ; use v2
; SI-NEXT: ;;#ASMEND
@@ -7151,11 +7200,11 @@ define amdgpu_kernel void @v_fneg_inlineasm_f32(ptr addrspace(1) %out, ptr addrs
; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: v_add_u32_e32 v2, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT: v_mov_b32_e32 v5, s7
; VI-NEXT: flat_load_dword v6, v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_load_dword v2, v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v5, s7
; VI-NEXT: v_add_u32_e32 v0, vcc, s6, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; VI-NEXT: flat_load_dword v0, v[0:1] glc
@@ -7163,7 +7212,8 @@ define amdgpu_kernel void @v_fneg_inlineasm_f32(ptr addrspace(1) %out, ptr addrs
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: v_mul_f32_e64 v2, v6, -v2
+; VI-NEXT: v_mul_f32_e32 v2, v6, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; VI-NEXT: ;;#ASMSTART
; VI-NEXT: ; use v2
; VI-NEXT: ;;#ASMEND
@@ -7644,9 +7694,9 @@ define amdgpu_kernel void @free_fold_src_code_size_cost_use_f32_nsz(ptr addrspac
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: v_fma_f32 v2, v8, -v2, -2.0
-; SI-NEXT: v_mul_f32_e32 v3, v2, v3
-; SI-NEXT: v_mul_f32_e32 v2, v2, v4
+; SI-NEXT: v_fma_f32 v2, v8, v2, 2.0
+; SI-NEXT: v_mul_f32_e64 v3, -v2, v3
+; SI-NEXT: v_mul_f32_e64 v2, -v2, v4
; SI-NEXT: flat_store_dword v[0:1], v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: flat_store_dword v[0:1], v2
@@ -7681,9 +7731,9 @@ define amdgpu_kernel void @free_fold_src_code_size_cost_use_f32_nsz(ptr addrspac
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
-; VI-NEXT: v_fma_f32 v2, v8, -v2, -2.0
-; VI-NEXT: v_mul_f32_e32 v3, v2, v3
-; VI-NEXT: v_mul_f32_e32 v2, v2, v4
+; VI-NEXT: v_fma_f32 v2, v8, v2, 2.0
+; VI-NEXT: v_mul_f32_e64 v3, -v2, v3
+; VI-NEXT: v_mul_f32_e64 v2, -v2, v4
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -8045,8 +8095,8 @@ define float @denorm_snan_fmul_neg1_to_fneg(float %x, float %y) {
; GCN-LABEL: denorm_snan_fmul_neg1_to_fneg:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v0
-; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v0
+; GCN-NEXT: v_mul_f32_e64 v0, -v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
%canonical = fmul float %x, %x
%mul = fmul float %canonical, -1.0
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
index 3cbde439cdf99..6c28c499dcb7d 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
@@ -777,7 +777,8 @@ define float @v_fneg_mul_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
@@ -802,8 +803,9 @@ define { float, float } @v_fneg_mul_multi_use_mul_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_multi_use_mul_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v1
-; GCN-NEXT: v_mul_f32_e32 v1, -4.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
@@ -817,7 +819,8 @@ define float @v_fneg_mul_fneg_x_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_fneg_x_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_f32_e64 v0, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%mul = fmul float %fneg.a, %b
@@ -829,7 +832,8 @@ define float @v_fneg_mul_x_fneg_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_x_fneg_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.b = fneg float %b
%mul = fmul float %a, %fneg.b
@@ -841,7 +845,8 @@ define float @v_fneg_mul_fneg_fneg_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_fneg_fneg_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%fneg.b = fneg float %b
@@ -855,7 +860,8 @@ define { float, float } @v_fneg_mul_store_use_fneg_x_f32(float %a, float %b) #0
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
-; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_f32_e64 v0, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: v_mov_b32_e32 v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
@@ -870,7 +876,8 @@ define { float, float } @v_fneg_mul_multi_use_fneg_x_f32(float %a, float %b, flo
; GCN-LABEL: v_fneg_mul_multi_use_fneg_x_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e32 v3, v0, v1
+; GCN-NEXT: v_mul_f32_e64 v1, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v3, 0x80000000, v1
; GCN-NEXT: v_mul_f32_e64 v1, -v0, v2
; GCN-NEXT: v_mov_b32_e32 v0, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -2570,7 +2577,8 @@ define float @v_fneg_fma_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, -v1, -v2
+; GCN-NEXT: v_fma_f32 v0, v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz float @llvm.fma.f32(float %a, float %b, float %c)
%fneg = fneg float %fma
@@ -2611,8 +2619,9 @@ define { float, float } @v_fneg_fma_multi_use_fma_f32_nsz(float %a, float %b, fl
; GCN-LABEL: v_fneg_fma_multi_use_fma_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, -v1, -v2
-; GCN-NEXT: v_mul_f32_e32 v1, -4.0, v0
+; GCN-NEXT: v_fma_f32 v1, v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz float @llvm.fma.f32(float %a, float %b, float %c)
%fneg = fneg float %fma
@@ -2639,7 +2648,8 @@ define float @v_fneg_fma_fneg_x_y_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_fneg_x_y_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, v1, -v2
+; GCN-NEXT: v_fma_f32 v0, -v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%fma = call nsz float @llvm.fma.f32(float %fneg.a, float %b, float %c)
@@ -2664,7 +2674,8 @@ define float @v_fneg_fma_x_fneg_y_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_x_fneg_y_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, v1, -v2
+; GCN-NEXT: v_fma_f32 v0, v0, -v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.b = fneg float %b
%fma = call nsz float @llvm.fma.f32(float %a, float %fneg.b, float %c)
@@ -2690,7 +2701,8 @@ define float @v_fneg_fma_fneg_fneg_y_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_fneg_fneg_y_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, -v1, -v2
+; GCN-NEXT: v_fma_f32 v0, v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%fneg.b = fneg float %b
@@ -2717,7 +2729,8 @@ define float @v_fneg_fma_fneg_x_fneg_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_fneg_x_fneg_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, v1, v2
+; GCN-NEXT: v_fma_f32 v0, -v0, v1, -v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%fneg.c = fneg float %c
@@ -2743,7 +2756,8 @@ define float @v_fneg_fma_x_y_fneg_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fma_x_y_fneg_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, -v1, v2
+; GCN-NEXT: v_fma_f32 v0, v0, v1, -v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.c = fneg float %c
%fma = call nsz float @llvm.fma.f32(float %a, float %b, float %fneg.c)
@@ -2773,7 +2787,8 @@ define { float, float } @v_fneg_fma_store_use_fneg_x_y_f32_nsz(float %a, float %
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_xor_b32_e32 v3, 0x80000000, v0
-; GCN-NEXT: v_fma_f32 v0, v0, v1, -v2
+; GCN-NEXT: v_fma_f32 v0, -v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: v_mov_b32_e32 v1, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
@@ -2806,7 +2821,8 @@ define { float, float } @v_fneg_fma_multi_use_fneg_x_y_f32_nsz(float %a, float %
; GCN-LABEL: v_fneg_fma_multi_use_fneg_x_y_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v2, v0, v1, -v2
+; GCN-NEXT: v_fma_f32 v1, -v0, v1, v2
+; GCN-NEXT: v_xor_b32_e32 v2, 0x80000000, v1
; GCN-NEXT: v_mul_f32_e64 v1, -v0, v3
; GCN-NEXT: v_mov_b32_e32 v0, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -2839,7 +2855,7 @@ define float @v_fneg_fmad_f32_nsz(float %a, float %b, float %c) #0 {
; GCN-LABEL: v_fneg_fmad_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mad_f32 v0, v0, -v1, -v2
+; GCN-NEXT: v_mad_f32 v0, -v0, v1, -v2
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz float @llvm.fmuladd.f32(float %a, float %b, float %c)
%fneg = fneg float %fma
@@ -2868,10 +2884,10 @@ define <4 x float> @v_fneg_fmad_v4f32_nsz(<4 x float> %a, <4 x float> %b, <4 x f
; GCN-LABEL: v_fneg_fmad_v4f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mad_f32 v0, v0, -v4, -v8
-; GCN-NEXT: v_mad_f32 v1, v1, -v5, -v9
-; GCN-NEXT: v_mad_f32 v2, v2, -v6, -v10
-; GCN-NEXT: v_mad_f32 v3, v3, -v7, -v11
+; GCN-NEXT: v_mad_f32 v0, -v0, v4, -v8
+; GCN-NEXT: v_mad_f32 v1, -v1, v5, -v9
+; GCN-NEXT: v_mad_f32 v2, -v2, v6, -v10
+; GCN-NEXT: v_mad_f32 v3, -v3, v7, -v11
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
%fneg = fneg <4 x float> %fma
@@ -2898,7 +2914,7 @@ define { float, float } @v_fneg_fmad_multi_use_fmad_f32_nsz(float %a, float %b,
; GCN-LABEL: v_fneg_fmad_multi_use_fmad_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mad_f32 v0, v0, -v1, -v2
+; GCN-NEXT: v_mad_f32 v0, -v0, v1, -v2
; GCN-NEXT: v_mul_f32_e32 v1, -4.0, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma = call nsz float @llvm.fmuladd.f32(float %a, float %b, float %c)
@@ -3246,9 +3262,9 @@ define float @v_negated_rcp_f32(float %arg0, float %arg1) #1 {
; GCN-LABEL: v_negated_rcp_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v0, v0, v1, 2.0
+; GCN-NEXT: v_fma_f32 v0, -v0, v1, -2.0
; GCN-NEXT: v_rcp_f32_e32 v0, v0
-; GCN-NEXT: v_add_f32_e32 v0, v1, v0
+; GCN-NEXT: v_sub_f32_e32 v0, v1, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%neg.arg0 = fneg float %arg0
%fma = call nsz float @llvm.fma.f32(float %neg.arg0, float %arg1, float -2.0)
@@ -3265,7 +3281,8 @@ define float @v_fneg_mul_legacy_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_legacy_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
%fneg = fneg float %mul
@@ -3290,8 +3307,9 @@ define { float, float } @v_fneg_mul_legacy_multi_use_mul_legacy_f32(float %a, fl
; GCN-LABEL: v_fneg_mul_legacy_multi_use_mul_legacy_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
-; GCN-NEXT: v_mul_legacy_f32_e64 v1, -v0, 4.0
+; GCN-NEXT: v_mul_legacy_f32_e32 v1, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v1
+; GCN-NEXT: v_mul_legacy_f32_e32 v1, 4.0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
%fneg = fneg float %mul
@@ -3305,7 +3323,8 @@ define float @v_fneg_mul_legacy_fneg_x_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_legacy_fneg_x_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_legacy_f32_e64 v0, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%mul = call float @llvm.amdgcn.fmul.legacy(float %fneg.a, float %b)
@@ -3317,7 +3336,8 @@ define float @v_fneg_mul_legacy_x_fneg_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_legacy_x_fneg_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.b = fneg float %b
%mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %fneg.b)
@@ -3329,7 +3349,8 @@ define float @v_fneg_mul_legacy_fneg_fneg_f32(float %a, float %b) #0 {
; GCN-LABEL: v_fneg_mul_legacy_fneg_fneg_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_mul_legacy_f32_e64 v0, -v0, -v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
%fneg.b = fneg float %b
@@ -3343,7 +3364,8 @@ define { float, float } @v_fneg_mul_legacy_store_use_fneg_x_f32(float %a, float
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
-; GCN-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_legacy_f32_e64 v0, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: v_mov_b32_e32 v1, v2
; GCN-NEXT: s_setpc_b64 s[30:31]
%fneg.a = fneg float %a
@@ -3359,7 +3381,8 @@ define { float, float } @v_fneg_mul_legacy_multi_use_fneg_x_f32(float %a, float
; GCN-LABEL: v_fneg_mul_legacy_multi_use_fneg_x_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_legacy_f32_e32 v3, v0, v1
+; GCN-NEXT: v_mul_legacy_f32_e64 v1, -v0, v1
+; GCN-NEXT: v_xor_b32_e32 v3, 0x80000000, v1
; GCN-NEXT: v_mul_legacy_f32_e64 v1, -v0, v2
; GCN-NEXT: v_mov_b32_e32 v0, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -3537,7 +3560,8 @@ define { float, float } @v_fneg_interp_p1_f32(float %a, float %b) #0 {
; SI-LABEL: v_fneg_interp_p1_f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_mul_f32_e64 v1, v0, -v1
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_xor_b32_e32 v1, 0x80000000, v0
; SI-NEXT: s_mov_b32 m0, 0
; SI-NEXT: v_interp_p1_f32 v0, v1, attr0.x
; SI-NEXT: v_interp_p1_f32 v1, v1, attr0.y
@@ -3546,7 +3570,8 @@ define { float, float } @v_fneg_interp_p1_f32(float %a, float %b) #0 {
; VI-LABEL: v_fneg_interp_p1_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v1, v0, -v1
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v1, 0x80000000, v0
; VI-NEXT: s_mov_b32 m0, 0
; VI-NEXT: v_interp_p1_f32_e32 v0, v1, attr0.x
; VI-NEXT: v_interp_p1_f32_e32 v1, v1, attr0.y
@@ -3564,7 +3589,8 @@ define { float, float } @v_fneg_interp_p2_f32(float %a, float %b) #0 {
; SI-LABEL: v_fneg_interp_p2_f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_mul_f32_e64 v2, v0, -v1
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; SI-NEXT: v_mov_b32_e32 v1, 4.0
; SI-NEXT: v_mov_b32_e32 v0, 4.0
; SI-NEXT: s_mov_b32 m0, 0
@@ -3575,7 +3601,8 @@ define { float, float } @v_fneg_interp_p2_f32(float %a, float %b) #0 {
; VI-LABEL: v_fneg_interp_p2_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v2, v0, -v1
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: v_xor_b32_e32 v2, 0x80000000, v0
; VI-NEXT: v_mov_b32_e32 v1, 4.0
; VI-NEXT: v_mov_b32_e32 v0, 4.0
; VI-NEXT: s_mov_b32 m0, 0
@@ -3664,7 +3691,8 @@ define float @v_fneg_inlineasm_f32(float %a, float %b, float %c, i32 %d) #0 {
; GCN-LABEL: v_fneg_inlineasm_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ; use v0
; GCN-NEXT: ;;#ASMEND
@@ -3778,9 +3806,9 @@ define { float, float } @free_fold_src_code_size_cost_use_f32_nsz(ptr addrspace(
; GCN-LABEL: free_fold_src_code_size_cost_use_f32_nsz:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_fma_f32 v1, v2, -v3, -2.0
-; GCN-NEXT: v_mul_f32_e32 v0, v1, v4
-; GCN-NEXT: v_mul_f32_e32 v1, v1, v5
+; GCN-NEXT: v_fma_f32 v1, v2, v3, 2.0
+; GCN-NEXT: v_mul_f32_e64 v0, -v1, v4
+; GCN-NEXT: v_mul_f32_e64 v1, -v1, v5
; GCN-NEXT: s_setpc_b64 s[30:31]
%fma0 = call nsz float @llvm.fma.f32(float %a, float %b, float 2.0)
%fneg.fma0 = fneg float %fma0
@@ -3897,8 +3925,8 @@ define float @denorm_snan_fmul_neg1_to_fneg(float %x, float %y) {
; GCN-LABEL: denorm_snan_fmul_neg1_to_fneg:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e64 v0, v0, -v0
-; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v0
+; GCN-NEXT: v_mul_f32_e64 v0, -v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
%canonical = fmul float %x, %x
%mul = fmul float %canonical, -1.0
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-fold.ll b/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-fold.ll
new file mode 100644
index 0000000000000..4980bc82919be
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-fold.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope %s
+
+; Verify that fneg is not folded into a VOP3 NEG source modifier on multi-source
+; instructions unless nnan is present. The NEG modifier only flips the sign of
+; its designated source operand; when NaN propagates from a DIFFERENT operand,
+; the modifier has no effect on the propagated NaN's sign bit, making the fold
+; incorrect without nnan.
+
+; ----------------------------------------------------------------------------
+; fmul: fneg(fmul(x,y)) -> fmul(-x, y) is UNSAFE without nnan
+; ----------------------------------------------------------------------------
+
+define float @fneg_fmul_f32(float %x, float %y) {
+; CHECK-LABEL: fneg_fmul_f32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f32_e32 v0, v0, v1
+; CHECK-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul float %x, %y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+define float @fneg_fmul_f32_nnan(float %x, float %y) {
+; CHECK-LABEL: fneg_fmul_f32_nnan:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f32_e64 v0, v0, -v1
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul nnan float %x, %y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+; ----------------------------------------------------------------------------
+; fma: fneg(fma(x,y,z)) -> fma(x,-y,-z) is UNSAFE without nnan
+; (src0 has no NEG, so NaN from src0 gets wrong sign)
+; ----------------------------------------------------------------------------
+
+define float @fneg_fma_f32(float %x, float %y, float %z) {
+; CHECK-LABEL: fneg_fma_f32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_fma_f32 v0, v0, v1, v2
+; CHECK-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nsz float @llvm.fma.f32(float %x, float %y, float %z)
+ %neg = fneg float %fma
+ ret float %neg
+}
+
+define float @fneg_fma_f32_nnan(float %x, float %y, float %z) {
+; CHECK-LABEL: fneg_fma_f32_nnan:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_fma_f32 v0, v0, -v1, -v2
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nnan nsz float @llvm.fma.f32(float %x, float %y, float %z)
+ %neg = fneg float %fma
+ ret float %neg
+}
+
+; ----------------------------------------------------------------------------
+; fadd: fneg(fadd(x,y)) -> fadd(-x,-y) is SAFE (NEG on both sources)
+; Should still fold even without nnan (but needs nsz).
+; ----------------------------------------------------------------------------
+
+define float @fneg_fadd_f32(float %x, float %y) {
+; CHECK-LABEL: fneg_fadd_f32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_sub_f32_e64 v0, -v0, v1
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %add = fadd nsz float %x, %y
+ %neg = fneg float %add
+ ret float %neg
+}
+
+; ----------------------------------------------------------------------------
+; rcp: fneg(rcp(x)) -> rcp(-x) is SAFE (single source, NEG always covers NaN)
+; Should still fold even without nnan.
+; ----------------------------------------------------------------------------
+
+define float @fneg_rcp_f32(float %x) {
+; CHECK-LABEL: fneg_rcp_f32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_rcp_f32_e64 v0, -v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %rcp = call float @llvm.amdgcn.rcp.f32(float %x)
+ %neg = fneg float %rcp
+ ret float %neg
+}
+
+; ----------------------------------------------------------------------------
+; f16 variants
+; ----------------------------------------------------------------------------
+
+define half @fneg_fmul_f16(half %x, half %y) {
+; CHECK-LABEL: fneg_fmul_f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f16_e32 v0, v0, v1
+; CHECK-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul half %x, %y
+ %neg = fneg half %mul
+ ret half %neg
+}
+
+define half @fneg_fmul_f16_nnan(half %x, half %y) {
+; CHECK-LABEL: fneg_fmul_f16_nnan:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f16_e64 v0, v0, -v1
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul nnan half %x, %y
+ %neg = fneg half %mul
+ ret half %neg
+}
+
+define half @fneg_fma_f16(half %x, half %y, half %z) {
+; CHECK-LABEL: fneg_fma_f16:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_fma_f16 v0, v0, v1, v2
+; CHECK-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nsz half @llvm.fma.f16(half %x, half %y, half %z)
+ %neg = fneg half %fma
+ ret half %neg
+}
+
+define half @fneg_fma_f16_nnan(half %x, half %y, half %z) {
+; CHECK-LABEL: fneg_fma_f16_nnan:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_fma_f16 v0, v0, -v1, -v2
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nnan nsz half @llvm.fma.f16(half %x, half %y, half %z)
+ %neg = fneg half %fma
+ ret half %neg
+}
+
+declare float @llvm.fma.f32(float, float, float)
+declare half @llvm.fma.f16(half, half, half)
+declare float @llvm.amdgcn.rcp.f32(float)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-preservation.ll b/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-preservation.ll
new file mode 100644
index 0000000000000..15e3d5925e22a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fneg-nan-sign-preservation.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+
+; Verify that combines do not fold fneg into VOP3 NEG source modifiers when
+; NaN may be present. On AMDGPU, when the hardware propagates a NaN from one
+; source operand, a NEG modifier on a different operand does not flip the
+; propagated NaN's sign bit. IEEE 754 requires fneg to always flip the sign,
+; including for NaN values.
+
+; ============================================================================
+; performFNegCombine -- FMUL arm
+; fneg(fmul(x, y)) must NOT fold to fmul(x, fneg(y)) without nnan.
+; ============================================================================
+
+define float @fneg_fmul_f32_no_nnan(float %x, float %y) {
+; GFX9-LABEL: fneg_fmul_f32_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul float %x, %y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+define float @fneg_fmul_f32_nnan(float %x, float %y) {
+; GFX9-LABEL: fneg_fmul_f32_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul nnan float %x, %y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+; Multi-use: fmul result used by both fneg and another consumer.
+; Without nnan, the fmul must not have its sign absorbed.
+define { float, float } @fneg_fmul_f32_multi_use_no_nnan(float %x, float %y) {
+; GFX9-LABEL: fneg_fmul_f32_multi_use_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80000000, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %mul = fmul float %x, %y
+ %neg = fneg float %mul
+ %use = fmul float %mul, 4.0
+ %r0 = insertvalue { float, float } poison, float %neg, 0
+ %r1 = insertvalue { float, float } %r0, float %use, 1
+ ret { float, float } %r1
+}
+
+; When one operand already has fneg, cancelling two fneg is always safe
+; (no new source modifier introduced).
+define float @fneg_fmul_fneg_x_f32(float %x, float %y) {
+; GFX9-LABEL: fneg_fmul_fneg_x_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e64 v0, -v0, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %neg.x = fneg float %x
+ %mul = fmul float %neg.x, %y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+define float @fneg_fmul_fneg_y_f32(float %x, float %y) {
+; GFX9-LABEL: fneg_fmul_fneg_y_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e64 v0, v0, -v1
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %neg.y = fneg float %y
+ %mul = fmul float %x, %neg.y
+ %neg = fneg float %mul
+ ret float %neg
+}
+
+; ============================================================================
+; performFNegCombine -- FMA arm
+; fneg(fma(x, y, z)) must NOT fold to fma(x, fneg(y), fneg(z)) without nnan.
+; ============================================================================
+
+define float @fneg_fma_f32_no_nnan(float %x, float %y, float %z) {
+; GFX9-LABEL: fneg_fma_f32_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nsz float @llvm.fma.f32(float %x, float %y, float %z)
+ %neg = fneg float %fma
+ ret float %neg
+}
+
+define float @fneg_fma_f32_nnan(float %x, float %y, float %z) {
+; GFX9-LABEL: fneg_fma_f32_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v0, -v1, -v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %fma = call nnan nsz float @llvm.fma.f32(float %x, float %y, float %z)
+ %neg = fneg float %fma
+ ret float %neg
+}
+
+; ============================================================================
+; performFSubCombine
+; (fsub (fadd a, a), c) must NOT fold to mad(a, 2.0, fneg(c)) without nnan.
+; (fsub c, (fadd a, a)) must NOT fold to mad(a, -2.0, c) without nnan.
+; ============================================================================
+
+define float @fsub_fadd_aa_c_no_nnan(float %a, float %c) #0 {
+; GFX9-LABEL: fsub_fadd_aa_c_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v0, 2.0, -v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sum = fadd contract float %a, %a
+ %sub = fsub contract float %sum, %c
+ ret float %sub
+}
+
+define float @fsub_fadd_aa_c_nnan(float %a, float %c) #0 {
+; GFX9-LABEL: fsub_fadd_aa_c_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v0, 2.0, -v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sum = fadd contract nnan float %a, %a
+ %sub = fsub contract nnan float %sum, %c
+ ret float %sub
+}
+
+define float @fsub_c_fadd_aa_no_nnan(float %a, float %c) #0 {
+; GFX9-LABEL: fsub_c_fadd_aa_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_sub_f32_e32 v0, v1, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sum = fadd contract float %a, %a
+ %sub = fsub contract float %c, %sum
+ ret float %sub
+}
+
+define float @fsub_c_fadd_aa_nnan(float %a, float %c) #0 {
+; GFX9-LABEL: fsub_c_fadd_aa_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v0, -2.0, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sum = fadd contract nnan float %a, %a
+ %sub = fsub contract nnan float %c, %sum
+ ret float %sub
+}
+
+; ============================================================================
+; performFMulCombine -- select-of-negative-powers-of-two arm
+; fmul x, (select y, -A, -B) must NOT fold to ldexp(fneg(x), ...) without nnan.
+; ============================================================================
+
+define double @fmul_select_neg_pow2_f64_no_nnan(double %x, i1 %cond) {
+; GFX9-LABEL: fmul_select_neg_pow2_f64_no_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v2, 2, 1, vcc
+; GFX9-NEXT: v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sel = select i1 %cond, double -2.0, double -4.0
+ %mul = fmul double %x, %sel
+ ret double %mul
+}
+
+define double @fmul_select_neg_pow2_f64_nnan(double %x, i1 %cond) {
+; GFX9-LABEL: fmul_select_neg_pow2_f64_nnan:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v2, 2, 1, vcc
+; GFX9-NEXT: v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %sel = select i1 %cond, double -2.0, double -4.0
+ %mul = fmul nnan double %x, %sel
+ ret double %mul
+}
+
+declare float @llvm.fma.f32(float, float, float)
+
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/fpext-free.ll b/llvm/test/CodeGen/AMDGPU/fpext-free.ll
index 10a925f9df687..51d37df83f7b5 100644
--- a/llvm/test/CodeGen/AMDGPU/fpext-free.ll
+++ b/llvm/test/CodeGen/AMDGPU/fpext-free.ll
@@ -722,38 +722,39 @@ define float @fsub_fpext_fneg_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
; GFX11-TRUE16-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, v0.l, -v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, v0 op_sel_hi:[0,1,1]
+; GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, -v0 op_sel_hi:[0,1,1]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
; GFX11-FAKE16: ; %bb.0: ; %entry
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, v0 op_sel_hi:[0,1,1]
+; GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, -v0 op_sel_hi:[0,1,1]
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-F32FLUSH-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
; GFX9-F32FLUSH: ; %bb.0: ; %entry
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-F32DENORM-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
; GFX9-F32DENORM: ; %bb.0: ; %entry
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
-; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e64 v0, -v0
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f32_f16_e64 v0, -v0
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -770,38 +771,39 @@ define float @fsub_fneg_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
; GFX11-TRUE16-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, v0.l, -v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, v0 op_sel_hi:[0,1,1]
+; GFX11-TRUE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, -v0 op_sel_hi:[0,1,1]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
; GFX11-FAKE16: ; %bb.0: ; %entry
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX11-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, v0 op_sel_hi:[0,1,1]
+; GFX11-FAKE16-NEXT: v_fma_mix_f32 v0, v2, -1.0, -v0 op_sel_hi:[0,1,1]
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-F32FLUSH-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
; GFX9-F32FLUSH: ; %bb.0: ; %entry
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-F32DENORM-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
; GFX9-F32DENORM: ; %bb.0: ; %entry
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
-; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e64 v0, -v0
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f32_f16_e64 v0, -v0
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index cf6732d30d080..da0bbbe572030 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -775,7 +775,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %o
; SI-STD-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-STD-NEXT: v_mad_f32 v2, v2, -v3, -v4
+; SI-STD-NEXT: v_mad_f32 v2, -v2, v3, -v4
; SI-STD-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-STD-NEXT: s_endpgm
;
@@ -815,8 +815,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %o
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e64 v2, v2, -v3
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v2, v4
+; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e64 v2, -v2, v4
; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMAF-NEXT: s_endpgm
%tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
@@ -857,8 +857,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1)
; SI-STD-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-STD-NEXT: v_mad_f32 v4, v2, -v3, -v4
-; SI-STD-NEXT: v_mad_f32 v2, v2, -v3, -v5
+; SI-STD-NEXT: v_mad_f32 v4, -v2, v3, -v4
+; SI-STD-NEXT: v_mad_f32 v2, -v2, v3, -v5
; SI-STD-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
@@ -909,9 +909,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1)
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e64 v2, v2, -v3
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v3, v2, v4
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v2, v5
+; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v2, v2, v3
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e64 v3, -v2, v4
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e64 v2, -v2, v5
; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index 417e880c95a17..aaecc47db880d 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -431,28 +431,31 @@ define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %s
; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
; SDAG-GFX9GEN: ; %bb.0:
; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v1, v0, v3
+; SDAG-GFX9GEN-NEXT: v_mov_b32_e32 v0, v1
; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
; SDAG-VI: ; %bb.0:
; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-VI-NEXT: v_mac_f32_e32 v1, v0, v3
+; SDAG-VI-NEXT: v_mov_b32_e32 v0, v1
; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
; SDAG-CI: ; %bb.0:
; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-CI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v0, v3
+; SDAG-CI-NEXT: v_mov_b32_e32 v0, v1
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negf16lo_f16lo_f16lo:
@@ -592,10 +595,12 @@ define float @v_mad_mix_f32_negabsf16lo_f16lo_f16lo(half %src0, half %src1, half
; SDAG-CI-LABEL: v_mad_mix_f32_negabsf16lo_f16lo_f16lo:
; SDAG-CI: ; %bb.0:
; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SDAG-CI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-CI-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; SDAG-CI-NEXT: v_mac_f32_e32 v1, v0, v3
+; SDAG-CI-NEXT: v_mov_b32_e32 v0, v1
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-CI-LABEL: v_mad_mix_f32_negabsf16lo_f16lo_f16lo:
@@ -2095,59 +2100,32 @@ define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, v2 op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; SDAG-GFX9GEN: ; %bb.0:
-; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
-; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; SDAG-CI: ; %bb.0:
-; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-CI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
-;
-; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; GISEL-GFX9GEN: ; %bb.0:
-; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e64 v3, -v0
-; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
-; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; GFX9GEN-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; GFX9GEN: ; %bb.0:
+; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9GEN-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; GISEL-VI: ; %bb.0:
-; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-VI-NEXT: v_cvt_f32_f16_e64 v3, -v0
-; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
-; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-CI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
-; GISEL-CI: ; %bb.0:
-; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_cvt_f32_f16_e64 v3, -v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GISEL-CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
+; CI-LABEL: v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; CI: ; %bb.0:
+; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%src0 = extractelement <2 x half> %src0.arg.bc, i32 0
%src0.neg = fneg half %src0
@@ -2331,19 +2309,19 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half
; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
; SDAG-GFX9GEN: ; %bb.0:
; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
; SDAG-VI: ; %bb.0:
; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v0, v3, v1
; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
@@ -2505,19 +2483,19 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo(i32 %src0.arg,
; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
; SDAG-GFX9GEN: ; %bb.0:
; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
; SDAG-VI: ; %bb.0:
; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v0, v3, v1
; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
diff --git a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract-legacy.ll b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract-legacy.ll
index 338c4eb5fd2f8..a3e1107ee6497 100644
--- a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract-legacy.ll
@@ -1,16 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GCN %s
; --------------------------------------------------------------------------------
; Don't fold if fneg can fold into the source
; --------------------------------------------------------------------------------
-; GCN-LABEL: {{^}}select_fneg_posk_src_rcp_legacy_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_rcp_legacy_f32_e64 [[RCP:v[0-9]+]], -[[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[RCP]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_rcp_legacy_f32(i32 %c) #2 {
+; GCN-LABEL: select_fneg_posk_src_rcp_legacy_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_load_dword s0, s[4:5], 0x9
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: v_rcp_legacy_f32_e64 v0, -v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -21,13 +33,22 @@ define amdgpu_kernel void @select_fneg_posk_src_rcp_legacy_f32(i32 %c) #2 {
ret void
}
-; GCN-LABEL: {{^}}select_fneg_posk_src_mul_legacy_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_mul_legacy_f32_e32 [[MUL:v[0-9]+]], -4.0, [[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[MUL]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_mul_legacy_f32(i32 %c) #2 {
+; GCN-LABEL: select_fneg_posk_src_mul_legacy_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_load_dword s0, s[4:5], 0x9
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: v_mul_legacy_f32_e64 v0, -v0, 4.0
+; GCN-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%mul = call float @llvm.amdgcn.fmul.legacy(float %x, float 4.0)
diff --git a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.ll b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.ll
index 71840f3387aff..eb97f8a40249b 100644
--- a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.ll
@@ -1,14 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
-; GCN-LABEL: {{^}}add_select_fabs_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN: v_add_f32_e64 v{{[0-9]+}}, |[[SELECT]]|, [[Z]]
define amdgpu_kernel void @add_select_fabs_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -21,16 +52,56 @@ define amdgpu_kernel void @add_select_fabs_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_use_lhs_fabs_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-; GCN: buffer_load_dword [[W:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, |[[SELECT]]|, [[Z]]
-; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, |[[X]]|, [[W]]
define amdgpu_kernel void @add_select_multi_use_lhs_fabs_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_use_lhs_fabs_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT: v_add_f32_e64 v1, |v1|, v2
+; SI-NEXT: v_add_f32_e64 v0, |v0|, v3
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_use_lhs_fabs_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT: v_add_f32_e64 v1, |v1|, v2
+; VI-NEXT: v_add_f32_e64 v0, |v0|, v3
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -46,18 +117,52 @@ define amdgpu_kernel void @add_select_multi_use_lhs_fabs_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_store_use_lhs_fabs_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN-DAG: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_add_f32_e64 [[ADD:v[0-9]+]], |[[SELECT]]|, [[Z]]
-; GCN-DAG: v_and_b32_e32 [[X_ABS:v[0-9]+]], 0x7fffffff, [[X]]
-
-; GCN: buffer_store_dword [[ADD]]
-; GCN: buffer_store_dword [[X_ABS]]
define amdgpu_kernel void @add_select_multi_store_use_lhs_fabs_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_store_use_lhs_fabs_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_store_use_lhs_fabs_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v3, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -71,16 +176,56 @@ define amdgpu_kernel void @add_select_multi_store_use_lhs_fabs_fabs_f32(i32 %c)
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_use_rhs_fabs_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-; GCN: buffer_load_dword [[W:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, |[[SELECT]]|, [[Z]]
-; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, |[[Y]]|, [[W]]
define amdgpu_kernel void @add_select_multi_use_rhs_fabs_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_use_rhs_fabs_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; SI-NEXT: v_add_f32_e64 v1, |v1|, v3
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_use_rhs_fabs_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_add_f32_e64 v0, |v0|, v2
+; VI-NEXT: v_add_f32_e64 v1, |v1|, v3
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -96,14 +241,46 @@ define amdgpu_kernel void @add_select_multi_use_rhs_fabs_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_var_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[Y]], |[[X]]|,
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_fabs_var_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_var_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v1, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_var_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v1, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -115,13 +292,42 @@ define amdgpu_kernel void @add_select_fabs_var_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -1.0, |[[X]]|,
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @add_select_fabs_negk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -1.0, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -1.0, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -132,12 +338,38 @@ define amdgpu_kernel void @add_select_fabs_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_negk_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -1.0, -2.0, s
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[X]], [[SELECT]]
define amdgpu_kernel void @add_select_fabs_negk_negk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_negk_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v1, -1.0, -2.0, s[0:1]
+; SI-NEXT: v_sub_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_negk_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v1, -1.0, -2.0, s[0:1]
+; VI-NEXT: v_sub_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, float -2.0, float -1.0
@@ -147,12 +379,38 @@ define amdgpu_kernel void @add_select_fabs_negk_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_posk_posk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 1.0, 2.0, s
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[X]]
define amdgpu_kernel void @add_select_posk_posk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_posk_posk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_posk_posk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, float 2.0, float 1.0
@@ -161,15 +419,42 @@ define amdgpu_kernel void @add_select_posk_posk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negk_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -1.0, |[[X]]|, [[VCC]]
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @add_select_negk_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negk_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -1.0, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negk_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -1.0, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -180,16 +465,44 @@ define amdgpu_kernel void @add_select_negk_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negliteralk_fabs_f32:
-; GCN-DAG: buffer_load_dword [[X:v[0-9]+]]
-; GCN-DAG: buffer_load_dword [[Y:v[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xc4800000
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[K]], |[[X]]|, [[VCC]]
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @add_select_negliteralk_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negliteralk_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0xc4800000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negliteralk_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0xc4800000
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v2, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -200,13 +513,42 @@ define amdgpu_kernel void @add_select_negliteralk_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_posk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 1.0, |[[X]]|, s{{\[[0-9]+:[0-9]+\]}}
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @add_select_fabs_posk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_posk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 1.0, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_posk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 1.0, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
@@ -218,15 +560,42 @@ define amdgpu_kernel void @add_select_fabs_posk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_posk_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 1.0, |[[X]]|, s{{\[[0-9]+:[0-9]+\]}}
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @add_select_posk_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_posk_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 1.0, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_posk_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 1.0, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -237,14 +606,46 @@ define amdgpu_kernel void @add_select_posk_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Z]], [[SELECT]]
define amdgpu_kernel void @add_select_fneg_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v2, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v2, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -257,16 +658,56 @@ define amdgpu_kernel void @add_select_fneg_fneg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_use_lhs_fneg_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-; GCN: buffer_load_dword [[W:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_sub_f32_e32 v{{[0-9]+}}, [[Z]], [[SELECT]]
-; GCN-DAG: v_sub_f32_e32 v{{[0-9]+}}, [[W]], [[X]]
define amdgpu_kernel void @add_select_multi_use_lhs_fneg_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_use_lhs_fneg_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v1, v2, v1
+; SI-NEXT: v_sub_f32_e32 v0, v3, v0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_use_lhs_fneg_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v1, v2, v1
+; VI-NEXT: v_sub_f32_e32 v0, v3, v0
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -282,18 +723,52 @@ define amdgpu_kernel void @add_select_multi_use_lhs_fneg_fneg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_store_use_lhs_fneg_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN-DAG: v_xor_b32_e32 [[NEG_X:v[0-9]+]], 0x80000000, [[X]]
-; GCN-DAG: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_sub_f32_e32 [[ADD:v[0-9]+]], [[Z]], [[SELECT]]
-
-; GCN: buffer_store_dword [[ADD]]
-; GCN: buffer_store_dword [[NEG_X]]
define amdgpu_kernel void @add_select_multi_store_use_lhs_fneg_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_store_use_lhs_fneg_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_xor_b32_e32 v3, 0x80000000, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v2, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_store_use_lhs_fneg_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v2, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v3, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -307,16 +782,56 @@ define amdgpu_kernel void @add_select_multi_store_use_lhs_fneg_fneg_f32(i32 %c)
ret void
}
-; GCN-LABEL: {{^}}add_select_multi_use_rhs_fneg_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-; GCN: buffer_load_dword [[W:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[Y]], [[X]], vcc
-; GCN-DAG: v_sub_f32_e32 v{{[0-9]+}}, [[Z]], [[SELECT]]
-; GCN-DAG: v_sub_f32_e32 v{{[0-9]+}}, [[W]], [[Y]]
define amdgpu_kernel void @add_select_multi_use_rhs_fneg_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_multi_use_rhs_fneg_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v2, v0
+; SI-NEXT: v_sub_f32_e32 v1, v3, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_multi_use_rhs_fneg_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v2, v0
+; VI-NEXT: v_sub_f32_e32 v1, v3, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -332,14 +847,46 @@ define amdgpu_kernel void @add_select_multi_use_rhs_fneg_fneg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_var_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[Y]], -[[X]],
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_fneg_var_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_var_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v1, -v0, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_var_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v1, -v0, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -351,13 +898,42 @@ define amdgpu_kernel void @add_select_fneg_var_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 1.0, [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_fneg_negk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, 1.0, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, 1.0, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -368,14 +944,44 @@ define amdgpu_kernel void @add_select_fneg_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_inv2pi_f32:
-; GCN-DAG: buffer_load_dword [[X:v[0-9]+]]
-; GCN-DAG: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xbe22f983
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[K]], [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_fneg_inv2pi_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_inv2pi_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_inv2pi_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -386,16 +992,43 @@ define amdgpu_kernel void @add_select_fneg_inv2pi_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_neginv2pi_f32:
-; GCN-DAG: buffer_load_dword [[X:v[0-9]+]]
-; GCN-DAG: buffer_load_dword [[Y:v[0-9]+]]
-; SI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e22f983
-
-; SI: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[K]], [[X]], vcc
-; VI: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 0.15915494, [[X]], vcc
-
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_fneg_neginv2pi_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_neginv2pi_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0x3e22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_neginv2pi_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, 0.15915494, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -406,13 +1039,38 @@ define amdgpu_kernel void @add_select_fneg_neginv2pi_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negk_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: s_cmp_eq_u32
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -1.0, -2.0, s
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[X]]
define amdgpu_kernel void @add_select_negk_negk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negk_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v1, -1.0, -2.0, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negk_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v1, -1.0, -2.0, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, float -2.0, float -1.0
@@ -421,15 +1079,42 @@ define amdgpu_kernel void @add_select_negk_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negliteralk_negliteralk_f32:
-; GCN-DAG: v_mov_b32_e32 [[K0:v[0-9]+]], 0xc5000000
-; GCN-DAG: v_mov_b32_e32 [[K1:v[0-9]+]], 0xc5800000
-; GCN-DAG: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: s_cmp_eq_u32
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], [[K1]], [[K0]], vcc
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[X]]
define amdgpu_kernel void @add_select_negliteralk_negliteralk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negliteralk_negliteralk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v1, 0xc5800000
+; SI-NEXT: v_mov_b32_e32 v2, 0xc5000000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-NEXT: v_add_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negliteralk_negliteralk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v1, 0xc5800000
+; VI-NEXT: v_mov_b32_e32 v2, 0xc5000000
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, float -2048.0, float -4096.0
@@ -438,12 +1123,38 @@ define amdgpu_kernel void @add_select_negliteralk_negliteralk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_negk_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 1.0, 2.0, s
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[X]]
define amdgpu_kernel void @add_select_fneg_negk_negk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_negk_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_negk_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, float -2.0, float -1.0
@@ -453,15 +1164,42 @@ define amdgpu_kernel void @add_select_fneg_negk_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negk_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 vcc, -1, 0
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 1.0, [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_negk_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negk_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, 1.0, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negk_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, 1.0, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -472,13 +1210,42 @@ define amdgpu_kernel void @add_select_negk_fneg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fneg_posk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], -1.0, [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_fneg_posk_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fneg_posk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, -1.0, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fneg_posk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, -1.0, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -489,15 +1256,42 @@ define amdgpu_kernel void @add_select_fneg_posk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_posk_fneg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 vcc, -1, 0
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], -1.0, [[X]], vcc
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Y]], [[SELECT]]
define amdgpu_kernel void @add_select_posk_fneg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_posk_fneg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_cndmask_b32_e32 v0, -1.0, v0, vcc
+; SI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_posk_fneg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_cndmask_b32_e32 v0, -1.0, v0, vcc
+; VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -508,14 +1302,46 @@ define amdgpu_kernel void @add_select_posk_fneg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negfabs_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], |[[Y]]|, -|[[X]]|,
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_negfabs_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negfabs_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, |v1|, -|v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negfabs_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, |v1|, -|v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -529,14 +1355,46 @@ define amdgpu_kernel void @add_select_negfabs_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -|[[Y]]|, |[[X]]|,
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_fabs_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -|v1|, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -|v1|, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -550,14 +1408,46 @@ define amdgpu_kernel void @add_select_fabs_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_neg_fabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], |[[Y]]|, -[[X]],
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_neg_fabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_neg_fabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, |v1|, -v0, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_neg_fabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, |v1|, -v0, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -570,14 +1460,46 @@ define amdgpu_kernel void @add_select_neg_fabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_fabs_neg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -[[Y]], |[[X]]|,
-; GCN: v_add_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Z]]
define amdgpu_kernel void @add_select_fabs_neg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_fabs_neg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -v1, |v0|, s[0:1]
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_fabs_neg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -v1, |v0|, s[0:1]
+; VI-NEXT: v_add_f32_e32 v0, v0, v2
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -590,14 +1512,46 @@ define amdgpu_kernel void @add_select_fabs_neg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_neg_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], |[[Y]]|, [[X]],
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Z]], [[SELECT]]
define amdgpu_kernel void @add_select_neg_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: add_select_neg_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, |v1|, v0, s[0:1]
+; SI-NEXT: v_sub_f32_e32 v0, v2, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_neg_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, |v1|, v0, s[0:1]
+; VI-NEXT: v_sub_f32_e32 v0, v2, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -611,14 +1565,46 @@ define amdgpu_kernel void @add_select_neg_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}add_select_negfabs_neg_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], |[[X]]|, [[Y]],
-; GCN: v_sub_f32_e32 v{{[0-9]+}}, [[Z]], [[SELECT]]
define amdgpu_kernel void @add_select_negfabs_neg_f32(i32 %c) #0 {
+; SI-LABEL: add_select_negfabs_neg_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, |v0|, v1, s[0:1]
+; SI-NEXT: v_sub_f32_e32 v0, v2, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: add_select_negfabs_neg_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, |v0|, v1, s[0:1]
+; VI-NEXT: v_sub_f32_e32 v0, v2, v0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
@@ -632,15 +1618,42 @@ define amdgpu_kernel void @add_select_negfabs_neg_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_posk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 4.0, -|[[X]]|, [[VCC]]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_posk_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_posk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 4.0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_posk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 4.0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -652,15 +1665,42 @@ define amdgpu_kernel void @mul_select_negfabs_posk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_posk_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 4.0, -|[[X]]|, [[VCC]]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_posk_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_posk_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 4.0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_posk_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 4.0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -672,13 +1712,42 @@ define amdgpu_kernel void @mul_select_posk_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_negk_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -4.0, -|[[X]]|, s{{\[[0-9]+:[0-9]+\]}}
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_negk_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_negk_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -4.0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_negk_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -4.0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -690,15 +1759,42 @@ define amdgpu_kernel void @mul_select_negfabs_negk_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negk_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], -4.0, -|[[X]]|, [[VCC]]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negk_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negk_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -4.0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negk_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, -4.0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -714,14 +1810,42 @@ define amdgpu_kernel void @mul_select_negk_negfabs_f32(i32 %c) #0 {
; Don't fold if fneg can fold into the source
; --------------------------------------------------------------------------------
-; GCN-LABEL: {{^}}select_fneg_posk_src_add_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_sub_f32_e32 [[ADD:v[0-9]+]], -4.0, [[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[ADD]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_add_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_add_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_sub_f32_e32 v0, -4.0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_add_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_sub_f32_e32 v0, -4.0, v0
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -732,13 +1856,38 @@ define amdgpu_kernel void @select_fneg_posk_src_add_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}select_fneg_posk_src_sub_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_sub_f32_e32 [[ADD:v[0-9]+]], 4.0, [[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[ADD]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_sub_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_sub_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_sub_f32_e32 v0, 4.0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_sub_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_sub_f32_e32 v0, 4.0, v0
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%add = fsub float %x, 4.0
@@ -748,13 +1897,38 @@ define amdgpu_kernel void @select_fneg_posk_src_sub_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}select_fneg_posk_src_mul_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], -4.0, [[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[MUL]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_mul_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_mul_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_mul_f32_e32 v0, -4.0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_mul_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_mul_f32_e32 v0, -4.0, v0
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
%mul = fmul float %x, 4.0
@@ -764,14 +1938,42 @@ define amdgpu_kernel void @select_fneg_posk_src_mul_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}select_fneg_posk_src_fma_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_fma_f32 [[FMA:v[0-9]+]], [[X]], -4.0, -[[Z]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[FMA]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_fma_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_fma_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_fma_f32 v0, v0, -4.0, -v1
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_fma_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_fma_f32 v0, v0, -4.0, -v1
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -782,13 +1984,43 @@ define amdgpu_kernel void @select_fneg_posk_src_fma_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}select_fneg_posk_src_fmad_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Z:v[0-9]+]]
-
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[X]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_fmad_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_fmad_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_fma_f32 v0, v0, -4.0, -v1
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_fmad_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_mul_f32_e32 v0, -4.0, v0
+; VI-NEXT: v_sub_f32_e32 v0, v0, v1
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%z = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -800,13 +2032,42 @@ define amdgpu_kernel void @select_fneg_posk_src_fmad_f32(i32 %c) #0 {
}
; FIXME: This one should fold to rcp
-; GCN-LABEL: {{^}}select_fneg_posk_src_rcp_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-
-; GCN: v_rcp_f32_e64 [[RCP:v[0-9]+]], -[[X]]
-; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 2.0, [[RCP]], vcc
-; GCN-NEXT: buffer_store_dword [[SELECT]]
define amdgpu_kernel void @select_fneg_posk_src_rcp_f32(i32 %c) #0 {
+; SI-LABEL: select_fneg_posk_src_rcp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: v_rcp_f32_e64 v0, -v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: select_fneg_posk_src_rcp_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_rcp_f32_e64 v0, -v0
+; VI-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -817,20 +2078,43 @@ define amdgpu_kernel void @select_fneg_posk_src_rcp_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_posk_inv2pi_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0
-; GCN-DAG: s_cselect_b64 [[VCC:.*]], -1, 0
-
-; SI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e22f983
-; SI: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[K]], -|[[X]]|, [[VCC]]
-; SI: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
-
-; VI: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 0.15915494, -|[[X]]|, [[VCC]]
-; VI: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_posk_inv2pi_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_posk_inv2pi_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0x3e22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_posk_inv2pi_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0.15915494, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -842,22 +2126,43 @@ define amdgpu_kernel void @mul_select_negfabs_posk_inv2pi_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_posk_inv2pi_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-
-; GCN-DAG: s_cselect_b64 [[VCC:.*]], -1, 0
-
-; SI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e22f983
-; SI: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[K]], -|[[X]]|, [[VCC]]
-; SI: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
-
-
-; VI: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 0.15915494, -|[[X]]|, [[VCC]]
-; VI: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_posk_inv2pi_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_posk_inv2pi_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0x3e22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_posk_inv2pi_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0.15915494, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -869,13 +2174,44 @@ define amdgpu_kernel void @mul_select_posk_inv2pi_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_negk_inv2pi_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xbe22f983
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[K]], -|[[X]]|, s
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_negk_inv2pi_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_negk_inv2pi_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_negk_inv2pi_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -887,16 +2223,44 @@ define amdgpu_kernel void @mul_select_negfabs_negk_inv2pi_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negk_inv2pi_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xbe22f983
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 s[0:1], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[K]], -|[[X]]|, s[0:1]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negk_inv2pi_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negk_inv2pi_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negk_inv2pi_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0xbe22f983
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -908,14 +2272,42 @@ define amdgpu_kernel void @mul_select_negk_inv2pi_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_posk_0_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 0, -|[[X]]|, [[VCC]]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_posk_0_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_posk_0_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_posk_0_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -928,15 +2320,42 @@ define amdgpu_kernel void @mul_select_negfabs_posk_0_f32(i32 %c) #0 {
}
-; GCN-LABEL: {{^}}mul_select_posk_0_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cselect_b64 [[VCC:.*]], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 0, -|[[X]]|, [[VCC]]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_posk_0_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_posk_0_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_posk_0_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -948,14 +2367,44 @@ define amdgpu_kernel void @mul_select_posk_0_negfabs_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negfabs_negk_0_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_bfrev_b32_e32 [[NEG0:v[0-9]+]], 1
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[NEG0]], -|[[X]]|, s{{\[[0-9]+:[0-9]+\]}}
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negfabs_negk_0_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negfabs_negk_0_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_bfrev_b32_e32 v2, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_eq_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negfabs_negk_0_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_bfrev_b32_e32 v2, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_eq_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -967,16 +2416,44 @@ define amdgpu_kernel void @mul_select_negfabs_negk_0_f32(i32 %c) #0 {
ret void
}
-; GCN-LABEL: {{^}}mul_select_negk_0_negfabs_f32:
-; GCN: buffer_load_dword [[X:v[0-9]+]]
-; GCN: buffer_load_dword [[Y:v[0-9]+]]
-
-; GCN: v_bfrev_b32_e32 [[NEG0:v[0-9]+]], 1
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 s[0:1], -1, 0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], [[NEG0]], -|[[X]]|, s[0:1]
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, [[SELECT]], [[Y]]
define amdgpu_kernel void @mul_select_negk_0_negfabs_f32(i32 %c) #0 {
+; SI-LABEL: mul_select_negk_0_negfabs_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_load_dword s0, s[4:5], 0x9
+; SI-NEXT: v_bfrev_b32_e32 v2, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mul_select_negk_0_negfabs_f32:
+; VI: ; %bb.0:
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_load_dword s0, s[4:5], 0x24
+; VI-NEXT: v_bfrev_b32_e32 v2, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, v2, -|v0|, s[0:1]
+; VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
%x = load volatile float, ptr addrspace(1) poison
%y = load volatile float, ptr addrspace(1) poison
%cmp = icmp eq i32 %c, 0
@@ -998,3 +2475,5 @@ declare float @llvm.amdgcn.fmul.legacy(float, float) #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.v2f16.ll b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.v2f16.ll
index c026a42993d48..51ac89b0f4c03 100644
--- a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.v2f16.ll
@@ -3505,17 +3505,22 @@ define <2 x half> @select_fneg_posk_src_mul_v2f16(<2 x i32> %c, <2 x half> %x) {
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; CI-NEXT: v_mov_b32_e32 v4, 0x4000
; CI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; CI-NEXT: v_mul_f32_e32 v2, -4.0, v2
-; CI-NEXT: v_mul_f32_e32 v3, -4.0, v3
-; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; CI-NEXT: v_mul_f32_e32 v3, 4.0, v3
; CI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; CI-NEXT: v_mul_f32_e32 v2, 4.0, v2
+; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; CI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; CI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -3861,26 +3866,31 @@ define <2 x half> @select_fneg_posk_src_fmad_v2f16_nsz(<2 x i32> %c, <2 x half>
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; CI-NEXT: v_mul_f32_e32 v2, -4.0, v2
-; CI-NEXT: v_mul_f32_e32 v4, -4.0, v4
-; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v4, v4
; CI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; CI-NEXT: v_mul_f32_e32 v4, 4.0, v4
+; CI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; CI-NEXT: v_mul_f32_e32 v2, 4.0, v2
+; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
; CI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; CI-NEXT: v_sub_f32_e32 v2, v2, v3
-; CI-NEXT: v_sub_f32_e32 v4, v4, v5
+; CI-NEXT: v_add_f32_e32 v4, v4, v5
+; CI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; CI-NEXT: v_add_f32_e32 v2, v2, v3
; CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v3, v4
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; CI-NEXT: v_mov_b32_e32 v4, 0x4000
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; CI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; CI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; CI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-fneg-constant.ll b/llvm/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
index be85016ad9bb5..57fdedbd14fdb 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
@@ -1,16 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GCN %s
; Test fcmp pred (fneg x), c -> fcmp (swapped pred) x, -c combine.
-; GCN-LABEL: {{^}}multi_use_fneg_src:
-; GCN: buffer_load_dword [[A:v[0-9]+]]
-; GCN: buffer_load_dword [[B:v[0-9]+]]
-; GCN: buffer_load_dword [[C:v[0-9]+]]
-
-; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], [[A]], [[B]]
-; GCN: v_cmp_eq_f32_e32 vcc, -4.0, [[MUL]]
-; GCN: buffer_store_dword [[MUL]]
define amdgpu_kernel void @multi_use_fneg_src() #0 {
+; GCN-LABEL: multi_use_fneg_src:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_cmp_eq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%b = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
@@ -25,15 +38,28 @@ define amdgpu_kernel void @multi_use_fneg_src() #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_foldable_use_fneg_src:
-; GCN: buffer_load_dword [[A:v[0-9]+]]
-; GCN: buffer_load_dword [[B:v[0-9]+]]
-; GCN: buffer_load_dword [[C:v[0-9]+]]
-
-; GCN: v_mul_f32_e32 [[MUL:v[0-9]+]], [[A]], [[B]]
-; GCN: v_cmp_eq_f32_e32 vcc, -4.0, [[A]]
-; GCN: v_mul_f32_e64 [[USE1:v[0-9]+]], [[MUL]], -[[MUL]]
define amdgpu_kernel void @multi_foldable_use_fneg_src() #0 {
+; GCN-LABEL: multi_foldable_use_fneg_src:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_cmp_eq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_mul_f32_e64 v1, v0, -v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%b = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
@@ -50,16 +76,28 @@ define amdgpu_kernel void @multi_foldable_use_fneg_src() #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_use_fneg:
-; GCN: buffer_load_dword [[A:v[0-9]+]]
-; GCN: buffer_load_dword [[B:v[0-9]+]]
-; GCN: buffer_load_dword [[C:v[0-9]+]]
-
-; GCN: v_mul_f32_e64 [[MUL:v[0-9]+]], [[A]], -[[B]]
-; GCN-NEXT: v_cmp_eq_f32_e32 vcc, 4.0, [[MUL]]
-; GCN-NOT: xor
-; GCN: buffer_store_dword [[MUL]]
define amdgpu_kernel void @multi_use_fneg() #0 {
+; GCN-LABEL: multi_use_fneg:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_cmp_eq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%b = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
@@ -74,15 +112,30 @@ define amdgpu_kernel void @multi_use_fneg() #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_foldable_use_fneg:
-; GCN: buffer_load_dword [[A:v[0-9]+]]
-; GCN: buffer_load_dword [[B:v[0-9]+]]
-
-; GCN: v_mul_f32_e32 [[MUL0:v[0-9]+]], [[A]], [[B]]
-; GCN: v_cmp_eq_f32_e32 vcc, -4.0, [[MUL0]]
-; GCN: v_mul_f32_e64 [[MUL1:v[0-9]+]], -[[MUL0]], [[MUL0]]
-; GCN: buffer_store_dword [[MUL1]]
define amdgpu_kernel void @multi_foldable_use_fneg() #0 {
+; GCN-LABEL: multi_foldable_use_fneg:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_cmp_eq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GCN-NEXT: v_mul_f32_e64 v0, -v0, v0
+; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%b = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
@@ -99,9 +152,22 @@ define amdgpu_kernel void @multi_foldable_use_fneg() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_oeq_posk_f32:
-; GCN: v_cmp_eq_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_oeq_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_oeq_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_eq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -112,9 +178,22 @@ define amdgpu_kernel void @test_setcc_fneg_oeq_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ogt_posk_f32:
-; GCN: v_cmp_gt_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ogt_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ogt_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_gt_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -125,9 +204,22 @@ define amdgpu_kernel void @test_setcc_fneg_ogt_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_oge_posk_f32:
-; GCN: v_cmp_ge_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_oge_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_oge_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_ge_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -138,9 +230,22 @@ define amdgpu_kernel void @test_setcc_fneg_oge_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_olt_posk_f32:
-; GCN: v_cmp_lt_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_olt_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_olt_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_lt_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -151,9 +256,22 @@ define amdgpu_kernel void @test_setcc_fneg_olt_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ole_posk_f32:
-; GCN: v_cmp_le_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ole_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ole_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_le_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -164,9 +282,22 @@ define amdgpu_kernel void @test_setcc_fneg_ole_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_one_posk_f32:
-; GCN: v_cmp_lg_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_one_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_one_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_lg_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -177,9 +308,22 @@ define amdgpu_kernel void @test_setcc_fneg_one_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ueq_posk_f32:
-; GCN: v_cmp_nlg_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ueq_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ueq_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_nlg_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -190,9 +334,22 @@ define amdgpu_kernel void @test_setcc_fneg_ueq_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ugt_posk_f32:
-; GCN: v_cmp_nle_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ugt_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ugt_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_nle_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -203,9 +360,22 @@ define amdgpu_kernel void @test_setcc_fneg_ugt_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_uge_posk_f32:
-; GCN: v_cmp_nlt_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_uge_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_uge_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -216,9 +386,22 @@ define amdgpu_kernel void @test_setcc_fneg_uge_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ult_posk_f32:
-; GCN: v_cmp_nge_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ult_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ult_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_nge_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -229,9 +412,22 @@ define amdgpu_kernel void @test_setcc_fneg_ult_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_ule_posk_f32:
-; GCN: v_cmp_ngt_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_ule_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_ule_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
@@ -242,9 +438,22 @@ define amdgpu_kernel void @test_setcc_fneg_ule_posk_f32() #0 {
ret void
}
-; GCN-LABEL: {{^}}test_setcc_fneg_une_posk_f32:
-; GCN: v_cmp_neq_f32_e32 vcc, -4.0, v{{[0-9]+}}
define amdgpu_kernel void @test_setcc_fneg_une_posk_f32() #0 {
+; GCN-LABEL: test_setcc_fneg_une_posk_f32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_cmp_neq_f32_e32 vcc, -4.0, v0
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
%a = load volatile float, ptr addrspace(1) poison
%x = load volatile i32, ptr addrspace(1) poison
%y = load volatile i32, ptr addrspace(1) poison
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac.ll b/llvm/test/CodeGen/AMDGPU/v_mac.ll
index 4658332ac395d..ee2eca16b4ed9 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac.ll
@@ -1,14 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mattr=+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign < %s | FileCheck --check-prefixes=SI,GCN %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global < %s | FileCheck --check-prefixes=VI-FLUSH,GCN %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN %s
-; GCN-LABEL: {{^}}mac_vvv:
-; GCN: buffer_load_dword [[A:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 glc{{$}}
-; GCN: buffer_load_dword [[B:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:4
-; GCN: buffer_load_dword [[C:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:8
-; GCN: v_mac_f32_e32 [[C]], [[A]], [[B]]
-; GCN: buffer_store_dword [[C]]
define amdgpu_kernel void @mac_vvv(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: mac_vvv:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[8:11], 0 offset:4 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mac_vvv:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dword v0, off, s[8:11], 0 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v1, off, s[8:11], 0 offset:4 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; VI-FLUSH-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -23,10 +61,28 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mad_inline_sgpr_inline:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]}}, s{{[0-9]+}}, 0.5, 0.5
define amdgpu_kernel void @mad_inline_sgpr_inline(ptr addrspace(1) %out, float %in) #0 {
+; SI-LABEL: mad_inline_sgpr_inline:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dword s6, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: v_mad_f32 v0, s6, 0.5, 0.5
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mad_inline_sgpr_inline:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dword s6, s[4:5], 0x2c
+; VI-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s3, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s2, -1
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: v_mad_f32 v0, s6, 0.5, 0.5
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%tmp0 = fmul float 0.5, %in
%tmp1 = fadd float %tmp0, 0.5
@@ -34,10 +90,44 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mad_vvs:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}
define amdgpu_kernel void @mad_vvs(ptr addrspace(1) %out, ptr addrspace(1) %in, float %c) #0 {
+; SI-LABEL: mad_vvs:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_load_dword s12, s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mad_f32 v0, v0, v1, s12
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mad_vvs:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_load_dword s12, s[4:5], 0x34
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_mad_f32 v0, v0, v1, s12
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
@@ -50,9 +140,44 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_ssv:
-; GCN: v_mac_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
define amdgpu_kernel void @mac_ssv(ptr addrspace(1) %out, ptr addrspace(1) %in, float %a) #0 {
+; SI-LABEL: mac_ssv:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_load_dword s12, s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mac_f32_e64 v0, s12, s12
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mac_ssv:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_load_dword s12, s[4:5], 0x34
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: buffer_load_dword v0, off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_mac_f32_e64 v0, s12, s12
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%c = load float, ptr addrspace(1) %in
@@ -62,10 +187,60 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_mad_same_add:
-; GCN: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]]
-; GCN: v_mac_f32_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}}
define amdgpu_kernel void @mac_mad_same_add(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: mac_mad_same_add:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:4 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[8:11], 0 offset:8 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[8:11], 0 offset:12 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[8:11], 0 offset:16 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mad_f32 v0, v0, v2, v1
+; SI-NEXT: v_mac_f32_e32 v1, v3, v4
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mac_mad_same_add:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dword v0, off, s[8:11], 0 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:4 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v1, off, s[8:11], 0 offset:8 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v3, off, s[8:11], 0 offset:12 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: buffer_load_dword v4, off, s[8:11], 0 offset:16 glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: v_mad_f32 v0, v0, v2, v1
+; VI-FLUSH-NEXT: v_mac_f32_e32 v1, v3, v4
+; VI-FLUSH-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -93,10 +268,43 @@ entry:
; There is no advantage to using v_mac when one of the operands is negated
; and v_mad accepts more operand types.
-; GCN-LABEL: {{^}}mad_neg_src0:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
define amdgpu_kernel void @mad_neg_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: mad_neg_src0:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mad_neg_src0:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_mad_f32 v0, -v0, v1, v2
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -113,10 +321,46 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}nsz_mad_sub0_src0:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
define amdgpu_kernel void @nsz_mad_sub0_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: nsz_mad_sub0_src0:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_sub_f32_e32 v0, 0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: nsz_mad_sub0_src0:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_sub_f32_e32 v0, 0, v0
+; VI-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; VI-FLUSH-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -133,10 +377,46 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}safe_mad_sub0_src0:
-; GCN: v_sub_f32_e32 [[SUB0:v[0-9]+]], 0,
-; GCN: v_ma{{[cd]}}_f32{{[_e32]*}} v{{[0-9]+}}, [[SUB0]], v{{[0-9]+}}
define amdgpu_kernel void @safe_mad_sub0_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: safe_mad_sub0_src0:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_sub_f32_e32 v0, 0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: safe_mad_sub0_src0:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_sub_f32_e32 v0, 0, v0
+; VI-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; VI-FLUSH-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -153,10 +433,43 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mad_neg_src1:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
define amdgpu_kernel void @mad_neg_src1(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: mad_neg_src1:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mad_f32 v0, v0, -v1, v2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mad_neg_src1:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_mad_f32 v0, v0, -v1, v2
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -173,10 +486,46 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}nsz_mad_sub0_src1:
-; GCN-NOT: v_mac_f32
-; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
define amdgpu_kernel void @nsz_mad_sub0_src1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: nsz_mad_sub0_src1:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_sub_f32_e32 v1, 0, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: nsz_mad_sub0_src1:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_sub_f32_e32 v1, 0, v1
+; VI-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; VI-FLUSH-NEXT: buffer_store_dword v2, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -193,10 +542,43 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mad_neg_src2:
-; GCN-NOT: v_mac
-; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}}
define amdgpu_kernel void @mad_neg_src2(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: mad_neg_src2:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mad_f32 v0, v0, v1, -v2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: mad_neg_src2:
+; VI-FLUSH: ; %bb.0: ; %entry
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_mov_b32 s7, 0xf000
+; VI-FLUSH-NEXT: s_mov_b32 s6, -1
+; VI-FLUSH-NEXT: s_mov_b32 s10, s6
+; VI-FLUSH-NEXT: s_mov_b32 s11, s7
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: s_mov_b32 s8, s2
+; VI-FLUSH-NEXT: s_mov_b32 s9, s3
+; VI-FLUSH-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
+; VI-FLUSH-NEXT: s_mov_b32 s4, s0
+; VI-FLUSH-NEXT: s_mov_b32 s5, s1
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_mad_f32 v0, v0, v1, -v2
+; VI-FLUSH-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-FLUSH-NEXT: s_endpgm
entry:
%b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1
%c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2
@@ -216,13 +598,59 @@ entry:
; Without special casing the inline constant check for v_mac_f32's
; src2, this fails to fold the 1.0 into a mad.
-; GCN-LABEL: {{^}}fold_inline_imm_into_mac_src2_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
-
-; GCN: v_add_f32_e32 [[TMP2:v[0-9]+]], [[A]], [[A]]
-; GCN: v_mad_f32 v{{[0-9]+}}, [[TMP2]], -4.0, 1.0
define amdgpu_kernel void @fold_inline_imm_into_mac_src2_f32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #3 {
+; SI-LABEL: fold_inline_imm_into_mac_src2_f32:
+; SI: ; %bb.0: ; %bb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT: v_add_f32_e32 v4, v2, v2
+; SI-NEXT: v_add_f32_e32 v5, v3, v3
+; SI-NEXT: v_mad_f32 v4, -v4, 4.0, 1.0
+; SI-NEXT: v_add_f32_e32 v3, v4, v3
+; SI-NEXT: v_mad_f32 v2, -v5, v2, 1.0
+; SI-NEXT: v_madmk_f32 v2, v2, 0x41000000, v3
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: fold_inline_imm_into_mac_src2_f32:
+; VI-FLUSH: ; %bb.0: ; %bb
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-FLUSH-NEXT: v_lshlrev_b32_e32 v4, 2, v0
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s3
+; VI-FLUSH-NEXT: v_add_u32_e32 v0, vcc, s2, v4
+; VI-FLUSH-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-FLUSH-NEXT: v_mov_b32_e32 v3, s5
+; VI-FLUSH-NEXT: v_add_u32_e32 v2, vcc, s4, v4
+; VI-FLUSH-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-FLUSH-NEXT: flat_load_dword v5, v[0:1] glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: flat_load_dword v2, v[2:3] glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_add_u32_e32 v0, vcc, s0, v4
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
+; VI-FLUSH-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-FLUSH-NEXT: v_add_f32_e32 v3, v5, v5
+; VI-FLUSH-NEXT: v_add_f32_e32 v4, v2, v2
+; VI-FLUSH-NEXT: v_mad_f32 v3, -v3, 4.0, 1.0
+; VI-FLUSH-NEXT: v_add_f32_e32 v2, v3, v2
+; VI-FLUSH-NEXT: v_mad_f32 v3, -v4, v5, 1.0
+; VI-FLUSH-NEXT: v_madmk_f32 v2, v3, 0x41000000, v2
+; VI-FLUSH-NEXT: flat_store_dword v[0:1], v2
+; VI-FLUSH-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -244,28 +672,81 @@ bb:
ret void
}
-; GCN-LABEL: {{^}}fold_inline_imm_into_mac_src2_f16:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_ushort [[B:v[0-9]+]]
-
-; SI-DAG: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], [[A]]
-; SI-DAG: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], [[B]]
-
-; SI: v_add_f32_e32 [[TMP2:v[0-9]+]], [[CVT_A]], [[CVT_A]]
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f16_f32
-; SI: v_add_f32
-; SI: v_add_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f16_f32
-
-; VI-FLUSH: v_add_f16_e32 [[TMP2:v[0-9]+]], [[A]], [[A]]
-; VI-FLUSH: v_mad_f16 v{{[0-9]+}}, [[TMP2]], -4.0, 1.0
define amdgpu_kernel void @fold_inline_imm_into_mac_src2_f16(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #3 {
+; SI-LABEL: fold_inline_imm_into_mac_src2_f16:
+; SI: ; %bb.0: ; %bb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: buffer_load_ushort v2, v[0:1], s[8:11], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_ushort v3, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_add_f32_e32 v4, v2, v2
+; SI-NEXT: v_add_f32_e32 v5, v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_mul_f32_e32 v4, 4.0, v4
+; SI-NEXT: v_mul_f32_e32 v2, v5, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_sub_f32_e32 v4, 1.0, v4
+; SI-NEXT: v_sub_f32_e32 v2, 1.0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v3, v4, v3
+; SI-NEXT: v_mul_f32_e32 v2, 0x41000000, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_add_f32_e32 v2, v3, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; VI-FLUSH-LABEL: fold_inline_imm_into_mac_src2_f16:
+; VI-FLUSH: ; %bb.0: ; %bb
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-FLUSH-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-FLUSH-NEXT: v_lshlrev_b32_e32 v4, 1, v0
+; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s3
+; VI-FLUSH-NEXT: v_add_u32_e32 v0, vcc, s2, v4
+; VI-FLUSH-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-FLUSH-NEXT: v_mov_b32_e32 v3, s5
+; VI-FLUSH-NEXT: v_add_u32_e32 v2, vcc, s4, v4
+; VI-FLUSH-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-FLUSH-NEXT: flat_load_ushort v5, v[0:1] glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: flat_load_ushort v2, v[2:3] glc
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
+; VI-FLUSH-NEXT: v_add_u32_e32 v0, vcc, s0, v4
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
+; VI-FLUSH-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-FLUSH-NEXT: v_add_f16_e32 v3, v5, v5
+; VI-FLUSH-NEXT: v_add_f16_e32 v4, v2, v2
+; VI-FLUSH-NEXT: v_mad_f16 v3, -v3, 4.0, 1.0
+; VI-FLUSH-NEXT: v_add_f16_e32 v2, v3, v2
+; VI-FLUSH-NEXT: v_mad_f16 v3, -v4, v5, 1.0
+; VI-FLUSH-NEXT: v_madmk_f16 v2, v3, 0x4800, v2
+; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
+; VI-FLUSH-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
@@ -287,29 +768,37 @@ bb:
ret void
}
-; Need to assume denormal handling is needed for dynamic denormal mode
-; GCN-LABEL: {{^}}v_mac_f32_dynamic:
-; GCN: v_mul_f32
-; GCN: v_add_f32
define float @v_mac_f32_dynamic(float %a, float %b, float %c) denormal_fpenv(float: dynamic|dynamic) {
+; GCN-LABEL: v_mac_f32_dynamic:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_add_f32_e32 v0, v0, v2
+; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%mad = fadd float %mul, %c
ret float %mad
}
-; GCN-LABEL: {{^}}v_mac_f32_dynamic_daz:
-; GCN: v_mul_f32
-; GCN: v_add_f32
define float @v_mac_f32_dynamic_daz(float %a, float %b, float %c) denormal_fpenv(float: preservesign|dynamic) {
+; GCN-LABEL: v_mac_f32_dynamic_daz:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_add_f32_e32 v0, v0, v2
+; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%mad = fadd float %mul, %c
ret float %mad
}
-; GCN-LABEL: {{^}}v_mac_f32_dynamic_ftz:
-; GCN: v_mul_f32
-; GCN: v_add_f32
define float @v_mac_f32_dynamic_ftz(float %a, float %b, float %c) denormal_fpenv(float: dynamic|preservesign) {
+; GCN-LABEL: v_mac_f32_dynamic_ftz:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT: v_add_f32_e32 v0, v0, v2
+; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%mad = fadd float %mul, %c
ret float %mad
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
index 8e1a4bf2a2d9f..20d683df7d3b2 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
@@ -1,23 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=VI %s
; FIXME: Can the SI case form the mac through the casts?
-; GCN-LABEL: {{^}}mac_f16:
-; GCN: {{buffer|flat}}_load_ushort v[[A_F16:[0-9]+]]
-; GCN: {{buffer|flat}}_load_ushort v[[B_F16:[0-9]+]]
-; GCN: {{buffer|flat}}_load_ushort v[[C_F16:[0-9]+]]
-
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_add_f32_e32
-
-; VI: v_mac_f16_e32 v[[C_F16]], v[[A_F16]], v[[B_F16]]
-; VI: buffer_store_short v[[C_F16]]
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16(
+; SI-LABEL: mac_f16:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: buffer_store_short v2, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -34,20 +83,106 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_same_add:
-; SI: v_mul_f32_e32
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_add_f32_e32
-; SI: v_add_f32_e32
-
-; VI: v_mad_f16 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]]
-; VI: v_mac_f16_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_same_add(
+; SI-LABEL: mac_f16_same_add:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x11
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x15
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_mov_b32 s7, s3
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s20, s12
+; SI-NEXT: s_mov_b32 s21, s13
+; SI-NEXT: s_mov_b32 s22, s2
+; SI-NEXT: s_mov_b32 s23, s3
+; SI-NEXT: s_mov_b32 s12, s14
+; SI-NEXT: s_mov_b32 s13, s15
+; SI-NEXT: s_mov_b32 s14, s2
+; SI-NEXT: s_mov_b32 s15, s3
+; SI-NEXT: s_mov_b32 s24, s18
+; SI-NEXT: s_mov_b32 s25, s19
+; SI-NEXT: s_mov_b32 s26, s2
+; SI-NEXT: s_mov_b32 s27, s3
+; SI-NEXT: buffer_load_ushort v0, off, s[4:7], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: buffer_load_ushort v2, off, s[20:23], 0
+; SI-NEXT: buffer_load_ushort v3, off, s[24:27], 0
+; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: s_mov_b32 s19, s3
+; SI-NEXT: buffer_load_ushort v4, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s0, s8
+; SI-NEXT: s_mov_b32 s1, s9
+; SI-NEXT: s_mov_b32 s4, s10
+; SI-NEXT: s_mov_b32 s5, s11
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v1, v2, v1
+; SI-NEXT: v_mul_f32_e32 v0, v3, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v1, v2
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v1, off, s[0:3], 0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_same_add:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x44
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x54
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s22, s2
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s20, s12
+; VI-NEXT: s_mov_b32 s21, s13
+; VI-NEXT: s_mov_b32 s23, s3
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s15
+; VI-NEXT: s_mov_b32 s14, s2
+; VI-NEXT: s_mov_b32 s15, s3
+; VI-NEXT: s_mov_b32 s24, s16
+; VI-NEXT: s_mov_b32 s25, s17
+; VI-NEXT: s_mov_b32 s26, s2
+; VI-NEXT: s_mov_b32 s27, s3
+; VI-NEXT: s_mov_b32 s16, s18
+; VI-NEXT: s_mov_b32 s17, s19
+; VI-NEXT: s_mov_b32 s18, s2
+; VI-NEXT: s_mov_b32 s19, s3
+; VI-NEXT: s_mov_b32 s6, s2
+; VI-NEXT: s_mov_b32 s7, s3
+; VI-NEXT: buffer_load_ushort v0, off, s[20:23], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[24:27], 0
+; VI-NEXT: buffer_load_ushort v3, off, s[4:7], 0
+; VI-NEXT: buffer_load_ushort v4, off, s[16:19], 0
+; VI-NEXT: s_mov_b32 s0, s8
+; VI-NEXT: s_mov_b32 s1, s9
+; VI-NEXT: s_mov_b32 s4, s10
+; VI-NEXT: s_mov_b32 s5, s11
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_mad_f16 v0, v0, v1, v2
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v4, v3
+; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_short v2, off, s[4:7], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r0,
ptr addrspace(1) %r1,
ptr addrspace(1) %a,
@@ -73,20 +208,69 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_a:
-; SI: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], v{{[0-9]+}}
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32 [[CVT_C:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16
-; SI: v_sub_f32
-
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_a(
+; SI-LABEL: mac_f16_neg_a:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e64 v1, -v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_a:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mad_f16 v0, -v0, v1, v2
+; VI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -104,20 +288,69 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_b:
-; SI: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], v{{[0-9]+}}
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32 [[CVT_C:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16
-; SI: v_sub_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_b(
+; SI-LABEL: mac_f16_neg_b:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_b:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mad_f16 v0, v0, -v1, v2
+; VI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -135,20 +368,69 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_c:
-; SI: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_c(
+; SI-LABEL: mac_f16_neg_c:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_sub_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_c:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mad_f16 v0, v0, v1, -v2
+; VI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -166,21 +448,71 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_a_safe_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}
-
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_add_f32
-; SI: v_cvt_f16_f32
-
-; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}}
-; VI: v_mac_f16_e32 v{{[0-9]+}}, v[[NEG_A]], v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_a_safe_fp_math(
+; SI-LABEL: mac_f16_neg_a_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_a_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v0, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: buffer_store_short v2, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -198,20 +530,71 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_b_safe_fp_math:
-
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_add_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}}
-; VI: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A]]
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_b_safe_fp_math(
+; SI-LABEL: mac_f16_neg_b_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_b_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_sub_f16_e32 v1, 0, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: buffer_store_short v2, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -229,19 +612,75 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_c_safe_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -v{{[0-9]}}
-; SI: v_add_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}}
-; VI: v_mac_f16_e32 v[[NEG_A]], v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_c_safe_fp_math(
+; SI-LABEL: mac_f16_neg_c_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e64 v1, -v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_c_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[4:7], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[16:19], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_sub_f16_e32 v1, 0, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v1, v0, v2
+; VI-NEXT: buffer_store_short v1, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -259,20 +698,71 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_a_nsz_fp_math:
-; SI: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], v{{[0-9]+}}
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32 [[CVT_C:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16
-; SI: v_sub_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_a_nsz_fp_math(
+; SI-LABEL: mac_f16_neg_a_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_a_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v0, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: buffer_store_short v2, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -290,19 +780,71 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_b_nsz_fp_math:
-; SI: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], v{{[0-9]+}}
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32 [[CVT_C:v[0-9]+]], v{{[0-9]+}}
-; SI: v_sub_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_b_nsz_fp_math(
+; SI-LABEL: mac_f16_neg_b_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_b_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_sub_f16_e32 v1, 0, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: buffer_store_short v2, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -320,20 +862,69 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_f16_neg_c_nsz_fp_math:
-; SI: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], v{{[0-9]+}}
-; SI: v_mul_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16_e32 [[CVT_C:v[0-9]+]], v{{[0-9]+}}
-; SI: v_cvt_f32_f16
-; SI: v_sub_f32_e32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_f16_neg_c_nsz_fp_math(
+; SI-LABEL: mac_f16_neg_c_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_ushort v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_ushort v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_ushort v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_sub_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_f16_neg_c_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0
+; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 0
+; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mad_f16 v0, v0, v1, -v2
+; VI-NEXT: buffer_store_short v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -351,27 +942,94 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16:
-; SI: v_mul_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_add_f32
-; SI: v_add_f32
-
-; VI: {{buffer|flat}}_load_dword v[[A_V2_F16:[0-9]+]]
-; VI: {{buffer|flat}}_load_dword v[[B_V2_F16:[0-9]+]]
-; VI: {{buffer|flat}}_load_dword v[[C_V2_F16:[0-9]+]]
-; VI-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
-; VI-DAG: v_mac_f16_sdwa v[[C_F16_1]], v[[A_V2_F16]], v[[B_V2_F16]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mac_f16_e32 v[[C_V2_F16]], v[[A_V2_F16]], v[[B_V2_F16]]
-; VI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[C_F16_1]]
-; VI-NOT: and
-; VI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[C_V2_F16]], v[[R_F16_HI]]
-
-; VI: {{buffer|flat}}_store_dword v[[R_V2_F16]]
-; VI: s_endpgm
define amdgpu_kernel void @mac_v2f16(
+; SI-LABEL: mac_v2f16:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_barrier
+; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_barrier
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_barrier
+; VI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_barrier
+; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; VI-NEXT: v_mac_f16_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_mac_f16_e32 v2, v0, v1
+; VI-NEXT: v_or_b32_e32 v0, v2, v3
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -390,23 +1048,138 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_same_add:
-; SI: v_mul_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f16_f32
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_add_f32
-; SI: v_add_f32
-
-; VI-DAG: v_mac_f16_sdwa v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mad_f16 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI-DAG: v_mac_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_same_add(
+; SI-LABEL: mac_v2f16_same_add:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x11
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x15
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s22, s2
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s20, s12
+; SI-NEXT: s_mov_b32 s21, s13
+; SI-NEXT: s_mov_b32 s23, s3
+; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_mov_b32 s7, s3
+; SI-NEXT: s_mov_b32 s12, s14
+; SI-NEXT: s_mov_b32 s13, s15
+; SI-NEXT: s_mov_b32 s14, s2
+; SI-NEXT: s_mov_b32 s15, s3
+; SI-NEXT: s_mov_b32 s24, s16
+; SI-NEXT: s_mov_b32 s25, s17
+; SI-NEXT: s_mov_b32 s26, s2
+; SI-NEXT: s_mov_b32 s27, s3
+; SI-NEXT: s_mov_b32 s16, s18
+; SI-NEXT: s_mov_b32 s17, s19
+; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s19, s3
+; SI-NEXT: buffer_load_dword v1, off, s[20:23], 0
+; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; SI-NEXT: buffer_load_dword v3, off, s[24:27], 0
+; SI-NEXT: buffer_load_dword v4, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s0, s8
+; SI-NEXT: s_mov_b32 s1, s9
+; SI-NEXT: s_mov_b32 s4, s10
+; SI-NEXT: s_mov_b32 s5, s11
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; SI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; SI-NEXT: v_mul_f32_e32 v1, v1, v2
+; SI-NEXT: v_mul_f32_e32 v2, v6, v7
+; SI-NEXT: v_mul_f32_e32 v0, v4, v0
+; SI-NEXT: v_mul_f32_e32 v4, v9, v5
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_add_f32_e32 v2, v2, v8
+; SI-NEXT: v_add_f32_e32 v1, v1, v3
+; SI-NEXT: v_add_f32_e32 v0, v0, v3
+; SI-NEXT: v_add_f32_e32 v3, v4, v8
+; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_same_add:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x44
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x54
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s22, s2
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s20, s12
+; VI-NEXT: s_mov_b32 s21, s13
+; VI-NEXT: s_mov_b32 s23, s3
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s15
+; VI-NEXT: s_mov_b32 s14, s2
+; VI-NEXT: s_mov_b32 s15, s3
+; VI-NEXT: s_mov_b32 s24, s16
+; VI-NEXT: s_mov_b32 s25, s17
+; VI-NEXT: s_mov_b32 s26, s2
+; VI-NEXT: s_mov_b32 s27, s3
+; VI-NEXT: s_mov_b32 s16, s18
+; VI-NEXT: s_mov_b32 s17, s19
+; VI-NEXT: s_mov_b32 s18, s2
+; VI-NEXT: s_mov_b32 s19, s3
+; VI-NEXT: s_mov_b32 s6, s2
+; VI-NEXT: s_mov_b32 s7, s3
+; VI-NEXT: buffer_load_dword v0, off, s[20:23], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[24:27], 0
+; VI-NEXT: buffer_load_dword v3, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v4, off, s[16:19], 0
+; VI-NEXT: s_mov_b32 s0, s8
+; VI-NEXT: s_mov_b32 s1, s9
+; VI-NEXT: s_mov_b32 s4, s10
+; VI-NEXT: s_mov_b32 s5, s11
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; VI-NEXT: v_mov_b32_e32 v7, v5
+; VI-NEXT: v_mac_f16_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT: v_mad_f16 v6, v0, v1, v2
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v5, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v7
+; VI-NEXT: v_mac_f16_e32 v2, v4, v3
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; VI-NEXT: v_or_b32_e32 v1, v6, v1
+; VI-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r0,
ptr addrspace(1) %r1,
ptr addrspace(1) %a,
@@ -432,19 +1205,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_a:
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_a(
+; SI-LABEL: mac_v2f16_neg_a:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_mul_f32_e32 v3, v4, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_a:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; VI-NEXT: v_mad_f16 v3, -v4, v3, v5
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_mad_f16 v0, -v1, v0, v2
+; VI-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -462,23 +1309,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_b
-; SI: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32_e32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32_e32
-
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_b(
+; SI-LABEL: mac_v2f16_neg_b:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_b:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; VI-NEXT: v_mad_f16 v3, v4, -v3, v5
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_mad_f16 v0, v1, -v0, v2
+; VI-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -496,22 +1413,92 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_c:
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f32_f16
-
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_c(
+; SI-LABEL: mac_v2f16_neg_c:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_sub_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_sub_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_c:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; VI-NEXT: v_mad_f16 v3, v4, v3, -v5
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_mad_f16 v0, v1, v0, -v2
+; VI-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -529,26 +1516,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_a_safe_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f32_f16
-; SI: v_add_f32
-; SI: v_cvt_f16_f32
-; SI: v_add_f32
-; SI: v_cvt_f16_f32
-
-; VI-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; VI-DAG: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}}
-; VI-DAG: v_sub_f16_sdwa v[[NEG_A0:[0-9]+]], [[ZERO]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mac_f16_sdwa v{{[0-9]+}}, v[[NEG_A0]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mac_f16_e32 v{{[0-9]+}}, v[[NEG_A1]], v{{[0-9]+}}
-
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_a_safe_fp_math(
+; SI-LABEL: mac_v2f16_neg_a_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e64 v3, -v3
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_a_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: buffer_load_dword v1, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v2, off, s[16:19], 0
+; VI-NEXT: v_mov_b32_e32 v3, 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v4, 0, v0
+; VI-NEXT: v_sub_f16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; VI-NEXT: v_mac_f16_e32 v1, v4, v2
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -566,24 +1620,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_b_safe_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_add_f32
-; SI: v_add_f32
-
-; VI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; VI: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}}
-; VI: v_sub_f16_sdwa v[[NEG_A0:[0-9]+]], [[ZERO]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mac_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-DAG: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A1]]
-
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_b_safe_fp_math(
+; SI-LABEL: mac_v2f16_neg_b_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e64 v3, -v3
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v3, v4, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_b_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: buffer_load_dword v1, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; VI-NEXT: v_mov_b32_e32 v3, 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v4, 0, v0
+; VI-NEXT: v_sub_f16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v3, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; VI-NEXT: v_mac_f16_e32 v1, v2, v4
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -601,22 +1724,95 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_c_safe_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_mul_f32
-; SI: v_add_f32
-; SI: v_add_f32
-
-; VI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
-; VI: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}}
-; VI: v_sub_f16_sdwa v[[NEG_A0:[0-9]+]], [[ZERO]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mac_f16_sdwa v[[NEG_A0]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-DAG: v_mac_f16_e32 v[[NEG_A1]], v{{[0-9]+}}, v{{[0-9]+}}
-
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_c_safe_fp_math(
+; SI-LABEL: mac_v2f16_neg_c_safe_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e64 v5, -v5
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e64 v2, -v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_c_safe_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: buffer_load_dword v0, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[16:19], 0
+; VI-NEXT: v_mov_b32_e32 v3, 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v4, 0, v0
+; VI-NEXT: v_sub_f16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v0, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; VI-NEXT: v_mac_f16_e32 v4, v1, v2
+; VI-NEXT: v_or_b32_e32 v0, v4, v0
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -634,23 +1830,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_a_nsz_fp_math:
-; SI: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_a_nsz_fp_math(
+; SI-LABEL: mac_v2f16_neg_a_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e64 v3, -v3
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_a_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: buffer_load_dword v1, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v2, off, s[16:19], 0
+; VI-NEXT: v_mov_b32_e32 v3, 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v4, 0, v0
+; VI-NEXT: v_sub_f16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; VI-NEXT: v_mac_f16_e32 v1, v4, v2
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -668,23 +1934,93 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_b_nsz_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
-; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_b_nsz_fp_math(
+; SI-LABEL: mac_v2f16_neg_b_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
+; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f16_f32_e64 v3, -v3
+; SI-NEXT: v_cvt_f16_f32_e64 v0, -v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_mul_f32_e32 v3, v4, v3
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v1, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_add_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_b_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: buffer_load_dword v1, off, s[4:7], 0
+; VI-NEXT: buffer_load_dword v2, off, s[12:15], 0
+; VI-NEXT: v_mov_b32_e32 v3, 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_sub_f16_e32 v4, 0, v0
+; VI-NEXT: v_sub_f16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mac_f16_sdwa v3, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; VI-NEXT: v_mac_f16_e32 v1, v2, v4
+; VI-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -702,23 +2038,92 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}mac_v2f16_neg_c_nsz_fp_math:
-; SI: v_cvt_f32_f16
-; SI: v_cvt_f32_f16
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_mul_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-; SI: v_sub_f32
-; SI: v_cvt_f16_f32
-
-; VI-NOT: v_mac_f16
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}}
-; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}}
-; GCN: s_endpgm
define amdgpu_kernel void @mac_v2f16_neg_c_nsz_fp_math(
+; SI-LABEL: mac_v2f16_neg_c_nsz_fp_math:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s18, s10
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; SI-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v1, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: v_sub_f32_e32 v1, v3, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-NEXT: v_sub_f32_e32 v0, v0, v2
+; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: mac_v2f16_neg_c_nsz_fp_math:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
+; VI-NEXT: s_mov_b32 s18, s10
+; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s16, s4
+; VI-NEXT: s_mov_b32 s17, s5
+; VI-NEXT: s_mov_b32 s14, s10
+; VI-NEXT: s_mov_b32 s12, s2
+; VI-NEXT: s_mov_b32 s13, s3
+; VI-NEXT: s_mov_b32 s15, s11
+; VI-NEXT: s_mov_b32 s4, s6
+; VI-NEXT: s_mov_b32 s5, s7
+; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s7, s11
+; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
+; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
+; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; VI-NEXT: v_mad_f16 v3, v4, v3, -v5
+; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_mad_f16 v0, v1, v0, -v2
+; VI-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a,
ptr addrspace(1) %b,
@@ -741,3 +2146,5 @@ declare void @llvm.amdgcn.s.barrier() #2
attributes #0 = { nounwind "no-signed-zeros-fp-math"="false" denormal_fpenv(preservesign) }
attributes #1 = { nounwind denormal_fpenv(preservesign) }
attributes #2 = { nounwind convergent }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
More information about the llvm-commits
mailing list