[llvm] [AMDGPU] Fix isCanonicalized to require canonical input for NaN-propa… (PR #208733)
Wooseok Lee via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 09:01:34 PDT 2026
https://github.com/wooseoklee updated https://github.com/llvm/llvm-project/pull/208733
>From 77e63da8af5f593c9742e8ff377d224a615de9d1 Mon Sep 17 00:00:00 2001
From: wooseoklee <wolee at amd.com>
Date: Thu, 30 Jul 2026 09:59:10 -0500
Subject: [PATCH] [AMDGPU] Fix isCanonicalized for IEEE-mode-dependent sNaN
quieting
isCanonicalized previously returned true unconditionally for most float ops,
allowing llvm.canonicalize to be silently elided even when the hardware
would not quiet sNaN. This causes incorrect code when a kernel runs with
MODE.IEEE=0.
Fix: introduce two helper booleans computed once at the top of each
isCanonicalized() overload:
IsGFX12Plus -- true when the target is GFX12 or newer.
QuietsSignalNaN -- true when sNaN quieting is guaranteed
(IsGFX12Plus || MODE.IEEE=1).
Ops are classified into three groups based on hardware testing:
Always quiet: v_cvt_f16_f32, v_cvt_pkrtz_f16_f32, v_dot2_f32_f16,
v_cvt_f32_ubyte{0-3} (integer->float, never produces NaN).
IEEE-dependent: all other float ops -- return QuietsSignalNaN.
Special case: DIV_SCALE passes sNaN through on pre-GFX12 even at
MODE.IEEE=1; return IsGFX12Plus.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 116 ++++++++++++------
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 5 +-
.../AMDGPU/fcanonicalize-elimination.ll | 20 +--
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 33 ++---
llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll | 14 ++-
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 116 ++++++++++--------
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 16 ++-
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 16 ++-
8 files changed, 213 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 403f5d7476dfc..bb4b13d1f1bdd 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15960,8 +15960,37 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
if (MaxDepth == 0)
return false;
+ // Prevent llvm.canonicalize from being silently dropped when the hardware
+ // won't quiet sNaN based on the following hardware sNaN quieting behavior.
+ //
+ // Pre-GFX12:
+ // Always quiet: v_cvt_f16_f32, v_cvt_pkrtz_f16_f32, v_dot2_f32_f16,
+ // v_cvt_f32_ubyte{0-3} (integer->float, never produces NaN).
+ // IEEE-dependent: all other float ops quiet sNaN only when MODE.IEEE=1.
+ // Never quiets: DIV_SCALE passes sNaN through even at MODE.IEEE=1.
+ //
+ // GFX12+: all exception-producing ops unconditionally quiet sNaN regardless
+ // of MODE.IEEE.
+ const bool IsGFX12Plus = Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12;
+ const bool QuietsSignalNaN =
+ IsGFX12Plus ||
+ DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>()->getMode().IEEE;
+
switch (Opcode) {
- // These will flush denorms if required.
+ // Unconditionally quiet sNaN (see comment above).
+ case ISD::FP_TO_FP16:
+ case AMDGPUISD::FP_TO_FP16:
+ case AMDGPUISD::CVT_PKRTZ_F16_F32:
+ case AMDGPUISD::CVT_F32_UBYTE0:
+ case AMDGPUISD::CVT_F32_UBYTE1:
+ case AMDGPUISD::CVT_F32_UBYTE2:
+ case AMDGPUISD::CVT_F32_UBYTE3:
+ return true;
+
+ // Passes sNaN through on pre-GFX12 even at ieee=1 (see comment above).
+ case AMDGPUISD::DIV_SCALE:
+ return IsGFX12Plus;
+
case ISD::FADD:
case ISD::FSUB:
case ISD::FMUL:
@@ -15975,7 +16004,6 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case ISD::FP_ROUND:
case ISD::FP_EXTEND:
case ISD::FP16_TO_FP:
- case ISD::FP_TO_FP16:
case ISD::BF16_TO_FP:
case ISD::FP_TO_BF16:
case ISD::FLDEXP:
@@ -15988,19 +16016,12 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::LOG:
case AMDGPUISD::EXP:
- case AMDGPUISD::DIV_SCALE:
case AMDGPUISD::DIV_FMAS:
case AMDGPUISD::DIV_FIXUP:
case AMDGPUISD::FRACT:
- case AMDGPUISD::CVT_PKRTZ_F16_F32:
- case AMDGPUISD::CVT_F32_UBYTE0:
- case AMDGPUISD::CVT_F32_UBYTE1:
- case AMDGPUISD::CVT_F32_UBYTE2:
- case AMDGPUISD::CVT_F32_UBYTE3:
- case AMDGPUISD::FP_TO_FP16:
case AMDGPUISD::SIN_HW:
case AMDGPUISD::COS_HW:
- return true;
+ return QuietsSignalNaN;
// It can/will be lowered or combined as a bit operation.
// Need to check their input recursively to handle.
@@ -16026,7 +16047,7 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case ISD::FSIN:
case ISD::FCOS:
case ISD::FSINCOS:
- return Op.getValueType().getScalarType() != MVT::f16;
+ return Op.getValueType().getScalarType() != MVT::f16 && QuietsSignalNaN;
case ISD::FMINNUM:
case ISD::FMAXNUM:
@@ -16036,17 +16057,20 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
case ISD::FMAXIMUMNUM:
- case AMDGPUISD::CLAMP:
case AMDGPUISD::FMED3:
case AMDGPUISD::FMAX3:
case AMDGPUISD::FMIN3:
case AMDGPUISD::FMAXIMUM3:
case AMDGPUISD::FMINIMUM3: {
+ // Only check denormals if sNaN quieting is guaranteed (see comment above).
+ if (!QuietsSignalNaN)
+ return false;
+ [[fallthrough]];
+ }
+ case AMDGPUISD::CLAMP: {
// FIXME: Shouldn't treat the generic operations different based these.
// However, we aren't really required to flush the result from
- // minnum/maxnum..
-
- // snans will be quieted, so we only need to worry about denormals.
+ // minnum/maxnum.
if (Subtarget->supportsMinMaxDenormModes() ||
// FIXME: denormalsEnabledForType is broken for dynamic
denormalsEnabledForType(DAG, Op.getValueType()))
@@ -16114,8 +16138,10 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
switch (IntrinsicID) {
case Intrinsic::amdgcn_cvt_pkrtz:
case Intrinsic::amdgcn_cubeid:
- case Intrinsic::amdgcn_frexp_mant:
case Intrinsic::amdgcn_fdot2:
+ return true;
+ case Intrinsic::amdgcn_div_scale:
+ return IsGFX12Plus;
case Intrinsic::amdgcn_rcp:
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_rsq_clamp:
@@ -16126,7 +16152,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case Intrinsic::amdgcn_log:
case Intrinsic::amdgcn_exp2:
case Intrinsic::amdgcn_sqrt:
- return true;
+ case Intrinsic::amdgcn_frexp_mant:
+ return QuietsSignalNaN;
default:
break;
}
@@ -16166,7 +16193,23 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
if (MaxDepth == 0)
return false;
+ // See the comment above the SDag isCanonicalized() for the hardware sNaN
+ // quieting behavior. The same rules apply here with GISel opcodes:
+ // G_AMDGPU_CVT_F32_UBYTE{0-3}: always canonical (integer->float, no NaN).
+ // amdgcn_cvt_pkrtz, amdgcn_cubeid, amdgcn_fdot2: always quiet sNaN.
+ // amdgcn_div_scale: passes sNaN through on pre-GFX12.
+ // All other ops: quiet sNaN only when MODE.IEEE=1 (or unconditionally on GFX12+).
+ const bool IsGFX12Plus = Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12;
+ const bool QuietsSignalNaN =
+ IsGFX12Plus || MF.getInfo<SIMachineFunctionInfo>()->getMode().IEEE;
+
switch (Opcode) {
+ case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
+ case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
+ case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
+ case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
+ return true;
+
case AMDGPU::G_FADD:
case AMDGPU::G_FSUB:
case AMDGPU::G_FMUL:
@@ -16189,11 +16232,8 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
case AMDGPU::G_FLOG10:
case AMDGPU::G_FPTRUNC:
case AMDGPU::G_AMDGPU_RCP_IFLAG:
- case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
- case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
- case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
- case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
- return true;
+ return QuietsSignalNaN;
+
case AMDGPU::G_FNEG:
case AMDGPU::G_FABS:
case AMDGPU::G_FCOPYSIGN:
@@ -16206,6 +16246,8 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
case AMDGPU::G_FMAXIMUM:
case AMDGPU::G_FMINIMUMNUM:
case AMDGPU::G_FMAXIMUMNUM: {
+ if (!QuietsSignalNaN)
+ return false;
if (Subtarget->supportsMinMaxDenormModes() ||
// FIXME: denormalsEnabledForType is broken for dynamic
denormalsEnabledForType(MRI.getType(Reg), MF))
@@ -16221,34 +16263,36 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
case AMDGPU::G_INTRINSIC:
case AMDGPU::G_INTRINSIC_CONVERGENT:
switch (cast<GIntrinsic>(MI)->getIntrinsicID()) {
+ case Intrinsic::amdgcn_cvt_pkrtz:
+ case Intrinsic::amdgcn_cubeid:
+ case Intrinsic::amdgcn_fdot2:
+ return true;
+ case Intrinsic::amdgcn_div_scale:
+ return IsGFX12Plus;
case Intrinsic::amdgcn_fmul_legacy:
case Intrinsic::amdgcn_fmad_ftz:
- case Intrinsic::amdgcn_sqrt:
case Intrinsic::amdgcn_fmed3:
case Intrinsic::amdgcn_sin:
case Intrinsic::amdgcn_cos:
- case Intrinsic::amdgcn_log:
- case Intrinsic::amdgcn_exp2:
case Intrinsic::amdgcn_log_clamp:
- case Intrinsic::amdgcn_rcp:
- case Intrinsic::amdgcn_rcp_legacy:
- case Intrinsic::amdgcn_rsq:
- case Intrinsic::amdgcn_rsq_clamp:
- case Intrinsic::amdgcn_rsq_legacy:
- case Intrinsic::amdgcn_div_scale:
case Intrinsic::amdgcn_div_fmas:
case Intrinsic::amdgcn_div_fixup:
case Intrinsic::amdgcn_fract:
- case Intrinsic::amdgcn_cvt_pkrtz:
- case Intrinsic::amdgcn_cubeid:
case Intrinsic::amdgcn_cubema:
case Intrinsic::amdgcn_cubesc:
case Intrinsic::amdgcn_cubetc:
- case Intrinsic::amdgcn_frexp_mant:
- case Intrinsic::amdgcn_fdot2:
case Intrinsic::amdgcn_trig_preop:
case Intrinsic::amdgcn_tanh:
- return true;
+ case Intrinsic::amdgcn_sqrt:
+ case Intrinsic::amdgcn_log:
+ case Intrinsic::amdgcn_exp2:
+ case Intrinsic::amdgcn_rcp:
+ case Intrinsic::amdgcn_rcp_legacy:
+ case Intrinsic::amdgcn_rsq:
+ case Intrinsic::amdgcn_rsq_clamp:
+ case Intrinsic::amdgcn_rsq_legacy:
+ case Intrinsic::amdgcn_frexp_mant:
+ return QuietsSignalNaN;
default:
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index 1e81c0e8f53ea..79b34f1bdb4f4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -494,8 +494,9 @@ define amdgpu_ps <2 x half> @fptrunc_v2f32_to_v2f16_div(<2 x float> %a) {
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: fptrunc_v2f32_to_v2f16_div:
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index 838042b79ae94..0ddec863ee201 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -578,8 +578,8 @@ define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_f64(ptr addrspace
; GCN-LABEL: test_fold_canonicalize_fmul_value_f32_no_ieee:
; GCN: v_mul_f32_e32 [[V:v[0-9]+]], 0x41700000, v{{[0-9]+}}
-; GCN-NOT: v_mul
-; GCN-NOT: v_max
+; VI-NEXT: v_mul_f32_e32 v{{[0-9]+}}, 1.0, [[V]]
+; GFX9-NEXT: v_max_f32_e32 [[V]], [[V]], [[V]]
; GCN-NEXT: ; return
define amdgpu_ps float @test_fold_canonicalize_fmul_value_f32_no_ieee(float %arg) {
entry:
@@ -590,8 +590,8 @@ entry:
; GCN-LABEL: test_fold_canonicalize_fmul_nnan_value_f32_no_ieee:
; GCN: v_mul_f32_e32 [[V:v[0-9]+]], 0x41700000, v{{[0-9]+}}
-; GCN-NOT: v_mul
-; GCN-NOT: v_max
+; VI-NEXT: v_mul_f32_e32 v{{[0-9]+}}, 1.0, [[V]]
+; GFX9-NEXT: v_max_f32_e32 [[V]], [[V]], [[V]]
; GCN-NEXT: ; return
define amdgpu_ps float @test_fold_canonicalize_fmul_nnan_value_f32_no_ieee(float %arg) {
entry:
@@ -602,8 +602,8 @@ entry:
; GCN-LABEL: {{^}}test_fold_canonicalize_fdiv_value_f32_no_ieee:
; GCN: v_div_fixup_f32
-; GCN-NOT: v_max
-; GCN-NOT: v_mul
+; VI-NEXT: v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
+; GFX9-NEXT: v_max_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
; GCN: ; return
define amdgpu_ps float @test_fold_canonicalize_fdiv_value_f32_no_ieee(float %arg0) {
entry:
@@ -684,17 +684,16 @@ define amdgpu_kernel void @test_fold_canonicalize_select_value_f32(ptr addrspace
; FIXME: canonicalize doens't work correctly without ieee_mode
; GCN-LABEL: {{^}}test_fold_canonicalize_minnum_value_no_ieee_mode:
-; GFX9-NOT: v0
-; GFX9-NOT: v1
; GFX9: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
; GFX9-NEXT: ; return to shader
; VI-FLUSH: v_min_f32_e32 v0, v0, v1
; VI-FLUSH-NEXT: v_mul_f32_e32 v0, 1.0, v0
; VI-FLUSH-NEXT: ; return
-; VI-DENORM-NOT: v0
; VI-DENORM: v_min_f32_e32 v0, v0, v1
+; VI-DENORM-NEXT: v_mul_f32_e32 v0, 1.0, v0
; VI-DENORM-NEXT: ; return
define amdgpu_ps float @test_fold_canonicalize_minnum_value_no_ieee_mode(float %arg0, float %arg1) {
%v = tail call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -721,7 +720,8 @@ define float @test_fold_canonicalize_minnum_value_ieee_mode(float %arg0, float %
; GCN-LABEL: {{^}}test_fold_canonicalize_minnum_value_no_ieee_mode_nnan:
; GCN: v_min_f32_e32 v0, v0, v1
; VI-FLUSH-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GCN-NEXT: ; return
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GCN: ; return
define amdgpu_ps float @test_fold_canonicalize_minnum_value_no_ieee_mode_nnan(float %arg0, float %arg1) #1 {
%v = tail call float @llvm.minnum.f32(float %arg0, float %arg1)
%canonicalized = tail call float @llvm.canonicalize.f32(float %v)
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 9db4987fc8210..f5a9ea2dd6b10 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -3733,7 +3733,7 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f32_sign_v2f16(<2 x float> inre
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, s1
; GFX9-NEXT: v_cvt_f16_f32_e32 v1, s0
; GFX9-NEXT: s_mov_b32 s0, 0x7fff7fff
-; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -3750,12 +3750,13 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f32_sign_v2f16(<2 x float> inre
;
; GFX11-FAKE16-LABEL: s_copysign_out_v2f16_mag_v2f32_sign_v2f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s1
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%mag.trunc = fptrunc <2 x float> %mag to <2 x half>
@@ -4153,7 +4154,7 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f16_sign_v2f32(<2 x half> inreg
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, s2
; GFX9-NEXT: v_cvt_f16_f32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s1, 0x7fff7fff
-; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: v_bfi_b32 v0, s1, v1, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
@@ -4170,12 +4171,13 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f16_sign_v2f32(<2 x half> inreg
;
; GFX11-FAKE16-LABEL: s_copysign_out_v2f16_mag_v2f16_sign_v2f32:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s2
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s1
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s1
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, s0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%sign.trunc = fptrunc <2 x float> %sign to <2 x half>
@@ -7116,7 +7118,7 @@ define amdgpu_ps i32 @s_copysign_v2f16_0_v2f32(<2 x float> inreg %sign) {
; GFX9: ; %bb.0:
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, s1
; GFX9-NEXT: v_cvt_f16_f32_e32 v1, s0
-; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0x80008000, v0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -7132,12 +7134,13 @@ define amdgpu_ps i32 @s_copysign_v2f16_0_v2f32(<2 x float> inreg %sign) {
;
; GFX11-FAKE16-LABEL: s_copysign_v2f16_0_v2f32:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s1
-; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, s0
+; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x80008000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%sign.trunc = fptrunc <2 x float> %sign to <2 x half>
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
index 791b9440c0683..817c88964af5b 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
@@ -1499,9 +1499,10 @@ define <2 x half> @v_fneg_minnum_multi_use_minnum_f16_no_ieee(half %a, half %b)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v1, 4.0, v0
-; GFX11-NEXT: v_pack_b32_f16 v0, -v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v1, 0x8000, v0
+; GFX11-NEXT: v_mul_f16_e32 v0, 4.0, v0
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX11-NEXT: s_setpc_b64 s[30:31]
%min = call half @llvm.minnum.f16(half %a, half %b)
%fneg = fneg half %min
@@ -1971,9 +1972,10 @@ define <2 x half> @v_fneg_maxnum_multi_use_maxnum_f16_no_ieee(half %a, half %b)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f16_e32 v1, 4.0, v0
-; GFX11-NEXT: v_pack_b32_f16 v0, -v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v1, 0x8000, v0
+; GFX11-NEXT: v_mul_f16_e32 v0, 4.0, v0
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX11-NEXT: s_setpc_b64 s[30:31]
%max = call half @llvm.maxnum.f16(half %a, half %b)
%fneg = fneg half %max
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index 2ab76c99095f8..3d877beedb51e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -156,9 +156,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_f32_to_v2f16_towardzero(float %a, f
;
; GISEL-LABEL: v_fptrunc_round_f32_to_v2f16_towardzero:
; GISEL: ; %bb.0:
-; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v0
-; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v1, v1, v0
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v1
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f32_to_v2f16_towardzero:
@@ -232,7 +230,8 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_constant_to_v2f16_towardzero(float
; GISEL-LABEL: v_fptrunc_round_constant_to_v2f16_towardzero:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v0
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, 1.0
+; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-NEXT: v_lshl_or_b32 v0, 0x3c00, 16, v0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_constant_to_v2f16_towardzero:
@@ -744,9 +743,10 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -796,9 +796,10 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -877,19 +878,24 @@ define amdgpu_gs void @v_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x
; GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v7, v3
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v3
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 1
+; GISEL-NEXT: v_cvt_f16_f32_e32 v7, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v2
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v3
-; GISEL-NEXT: v_pack_b32_f16 v3, v6, v7
-; GISEL-NEXT: v_pack_b32_f16 v1, v1, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v1, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
-; GISEL-NEXT: v_pk_add_f16 v0, v0, v3
-; GISEL-NEXT: v_pk_add_f16 v0, v1, v0
+; GISEL-NEXT: v_pk_add_f16 v0, v0, v1
+; GISEL-NEXT: v_pk_add_f16 v0, v2, v0
; GISEL-NEXT: global_store_dword v[4:5], v0, off
; GISEL-NEXT: s_endpgm
;
@@ -1315,12 +1321,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
; GISEL: ; %bb.0:
-; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
@@ -1375,12 +1382,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
; GISEL: ; %bb.0:
-; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
@@ -1440,12 +1448,14 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1506,12 +1516,14 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1587,18 +1599,22 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
; GISEL-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v6
; GISEL-NEXT: v_cvt_f16_f32_e32 v7, v7
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
-; GISEL-NEXT: v_pack_b32_f16 v2, v4, v5
-; GISEL-NEXT: v_pack_b32_f16 v3, v6, v7
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v6
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v3, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1683,18 +1699,22 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
; GISEL-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v6
; GISEL-NEXT: v_cvt_f16_f32_e32 v7, v7
-; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
-; GISEL-NEXT: v_pack_b32_f16 v2, v4, v5
-; GISEL-NEXT: v_pack_b32_f16 v3, v6, v7
+; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GISEL-NEXT: v_cvt_f16_f32_e32 v6, v6
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: v_or_b32_sdwa v3, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index bd2fa2a024fa6..1632214af0c8e 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -12934,6 +12934,7 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -13153,8 +13154,10 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX7-NEXT: v_max_f32_e32 v2, v3, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -13539,8 +13542,11 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX7-NEXT: v_max_f32_e32 v3, v4, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_alignbit_b32 v0, v3, v0, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -14055,9 +14061,13 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX7-NEXT: v_max_f32_e32 v3, v5, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_max_f32_e32 v0, v0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v3, v0, 16
; GFX7-NEXT: v_alignbit_b32 v1, v4, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index d5d49a74815fc..d82b54e0a4358 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -12967,6 +12967,7 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -13188,8 +13189,10 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX7-NEXT: v_min_f32_e32 v2, v3, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -13577,8 +13580,11 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX7-NEXT: v_min_f32_e32 v3, v4, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_alignbit_b32 v0, v3, v0, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -14096,9 +14102,13 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX7-NEXT: v_min_f32_e32 v3, v5, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_min_f32_e32 v0, v0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v3, v0, 16
; GFX7-NEXT: v_alignbit_b32 v1, v4, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
More information about the llvm-commits
mailing list