[llvm] [AMDGPU] Split the true16 fcopysign pattern by uniformity (PR #226898)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 00:02:05 PDT 2026
https://github.com/changpeng created https://github.com/llvm/llvm-project/pull/226898
A uniform 16-bit value already lives in the low half of an SGPR, so it can go straight to V_BFI_B32_e64. Only a divergent VGPR_16 operand needs the widening REG_SEQUENCE, which was ill-formed for the uniform case (operand wider than the lo16 slot it names) and confused DetectDeadLanes.
Fixes: ROCM-31212
>From 375a0e542e19ab9b4f3125b2759189171b194717 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Sun, 27 Sep 2026 23:57:22 -0700
Subject: [PATCH] [AMDGPU] Split the true16 fcopysign pattern by uniformity
A uniform 16-bit value already lives in the low half of an SGPR, so it can
go straight to V_BFI_B32_e64. Only a divergent VGPR_16 operand needs the
widening REG_SEQUENCE, which was ill-formed for the uniform case (operand
wider than the lo16 slot it names) and confused DetectDeadLanes.
Fixes: ROCM-31212
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 12 +-
llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll | 139 +++++---------
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 175 ++++++------------
llvm/test/CodeGen/AMDGPU/llvm.round.ll | 42 ++---
.../AMDGPU/true16-uniform-f16-phi-copysign.ll | 75 ++++++++
5 files changed, 205 insertions(+), 238 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/true16-uniform-f16-phi-copysign.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 3cf20ffb1fcf40..a40fe337c05649 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2464,10 +2464,16 @@ def : GCNPat <
}
let True16Predicate = UseRealTrue16Insts in {
def : GCNPat <
- (fcopysign fp16vt:$src0, fp16vt:$src1),
+ (UniformBinFrag<fcopysign> fp16vt:$src0, fp16vt:$src1),
+ (EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)),
+ SReg_32:$src0, SReg_32:$src1), lo16)
+>;
+
+def : GCNPat <
+ (DivergentBinFrag<fcopysign> fp16vt:$src0, fp16vt:$src1),
(EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)),
- (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16),
- (REG_SEQUENCE VGPR_32, $src1, lo16, (i16 (IMPLICIT_DEF)), hi16)), lo16)
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src0, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src1, lo16, (i16 (IMPLICIT_DEF)), hi16)), lo16)
>;
// TODO: Scalar case for 0 magnitude special case
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
index 31b868993156dd..251561a6c16293 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
@@ -361,26 +361,15 @@ define amdgpu_ps i32 @s_copysign_bf16_bf16(bfloat inreg %mag, bfloat inreg %sign
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11TRUE16-LABEL: s_copysign_bf16_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11FAKE16-LABEL: s_copysign_bf16_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_bf16_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
%cast = bitcast bfloat %op to i16
%zext = zext i16 %cast to i32
@@ -577,26 +566,15 @@ define amdgpu_ps i32 @s_copysign_bf16_f16(bfloat inreg %mag, half inreg %sign.f1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11TRUE16-LABEL: s_copysign_bf16_f16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11FAKE16-LABEL: s_copysign_bf16_f16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_bf16_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%sign = bitcast half %sign.f16 to bfloat
%op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
%cast = bitcast bfloat %op to i16
@@ -825,26 +803,15 @@ define amdgpu_ps i32 @s_copysign_f16_bf16(half inreg %mag, bfloat inreg %sign.bf
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11TRUE16-LABEL: s_copysign_f16_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11FAKE16-LABEL: s_copysign_f16_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_f16_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%sign = bitcast bfloat %sign.bf16 to half
%op = call half @llvm.copysign.f16(half %mag, half %sign)
%cast = bitcast half %op to i16
@@ -3133,41 +3100,23 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11TRUE16-LABEL: s_copysign_out_bf16_mag_f32_sign_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_bfe_u32 s2, s0, 0x10010
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e64 s3, s0, s0
-; GFX11TRUE16-NEXT: s_add_i32 s2, s2, s0
-; GFX11TRUE16-NEXT: s_bitset1_b32 s0, 22
-; GFX11TRUE16-NEXT: s_addk_i32 s2, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
-; GFX11TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11TRUE16-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11TRUE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
-; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11FAKE16-LABEL: s_copysign_out_bf16_mag_f32_sign_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_bfe_u32 s2, s0, 0x10010
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e64 s3, s0, s0
-; GFX11FAKE16-NEXT: s_add_i32 s2, s2, s0
-; GFX11FAKE16-NEXT: s_bitset1_b32 s0, 22
-; GFX11FAKE16-NEXT: s_addk_i32 s2, 0x7fff
-; GFX11FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11FAKE16-NEXT: s_cselect_b32 s0, s0, s2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11FAKE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_out_bf16_mag_f32_sign_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX11-NEXT: v_cmp_u_f32_e64 s3, s0, s0
+; GFX11-NEXT: s_add_i32 s2, s2, s0
+; GFX11-NEXT: s_bitset1_b32 s0, 22
+; GFX11-NEXT: s_addk_i32 s2, 0x7fff
+; GFX11-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, s0, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%mag.trunc = fptrunc float %mag to bfloat
%out = call bfloat @llvm.copysign.bf16(bfloat %mag.trunc, bfloat %sign)
%cast = bitcast bfloat %out to i16
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index f9cb5d91ff615d..dc75ba16b8bd11 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -39,22 +39,13 @@ define amdgpu_ps i16 @s_copysign_f16(half inreg %mag, half inreg %sign) {
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_copysign_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_copysign_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%out = call half @llvm.copysign.f16(half %mag, half %sign)
%cast = bitcast half %out to i16
ret i16 %cast
@@ -1214,106 +1205,55 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0x1ff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s3, s0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_bfe_u32 s3, s1, 0xb0014
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xffe
-; GFX11-TRUE16-NEXT: v_med3_i32 v0, s4, 0, 13
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: s_or_b32 s1, s0, 0x1000
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s5, s4
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, s1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_addk_i32 s3, 0xfc10
-; GFX11-TRUE16-NEXT: s_or_b32 s1, s5, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s3, 12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_or_b32 s4, s0, s4
-; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 2
-; GFX11-TRUE16-NEXT: s_or_b32 s4, s4, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s4
-; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
-; GFX11-TRUE16-NEXT: s_movk_i32 s4, 0x7e00
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, s1, 0x7c00
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s4, 0x7c00
-; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0x1ff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s3, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-FAKE16-NEXT: s_bfe_u32 s3, s1, 0xb0014
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffe
-; GFX11-FAKE16-NEXT: v_med3_i32 v0, s4, 0, 13
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s0, 0x1000
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, s4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s5, s4
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, s1
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-FAKE16-NEXT: s_addk_i32 s3, 0xfc10
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s5, s1
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 12
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s0, s4
-; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, s1, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
-; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 2
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s4
-; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
-; GFX11-FAKE16-NEXT: s_movk_i32 s4, 0x7e00
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, s1, 0x7c00
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s4, 0x7c00
-; GFX11-FAKE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_out_f16_mag_f64_sign_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s3, s1, 0x1ff
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s3, s0
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_bfe_u32 s3, s1, 0xb0014
+; GFX11-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-NEXT: s_sub_i32 s4, 0x3f1, s3
+; GFX11-NEXT: s_and_b32 s1, s1, 0xffe
+; GFX11-NEXT: v_med3_i32 v0, s4, 0, 13
+; GFX11-NEXT: s_or_b32 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_or_b32 s1, s0, 0x1000
+; GFX11-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_lshr_b32 s5, s1, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s4, s5, s4
+; GFX11-NEXT: s_cmp_lg_u32 s4, s1
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-NEXT: s_or_b32 s1, s5, s1
+; GFX11-NEXT: s_lshl_b32 s4, s3, 12
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s4, s0, s4
+; GFX11-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-NEXT: s_cselect_b32 s1, s1, s4
+; GFX11-NEXT: s_and_b32 s4, s1, 7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_gt_i32 s4, 5
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_eq_u32 s4, 3
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_lshr_b32 s1, s1, 2
+; GFX11-NEXT: s_or_b32 s4, s4, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s1, s1, s4
+; GFX11-NEXT: s_cmp_lt_i32 s3, 31
+; GFX11-NEXT: s_movk_i32 s4, 0x7e00
+; GFX11-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s0, s4, 0x7c00
+; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x40f
+; GFX11-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%mag.trunc = fptrunc double %mag to half
%result = call half @llvm.copysign.f16(half %mag.trunc, half %sign)
%cast = bitcast half %result to i16
@@ -2669,9 +2609,8 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f32_sign_f16(float inreg %mag, half
; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f32_sign_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
index 265cbcb9402733..9c7c0f0de3ad23 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
@@ -835,15 +835,14 @@ define amdgpu_kernel void @round_f16(ptr addrspace(1) %out, i32 %x.arg) #0 {
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_sub_f16_e32 v0.h, s2, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s3, |v0.h|, 0.5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 0x3c00, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
+; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -997,27 +996,26 @@ define amdgpu_kernel void @round_v2f16(ptr addrspace(1) %out, i32 %in.arg) #0 {
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, s2
-; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.h, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v1.l, s2, v0.l
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v1.h, s3, v0.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.h, s2
+; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, s3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v1.h, s2, v0.h
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v1.l, s3, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s6, |v1.l|, 0.5
; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s7, |v1.h|, 0.5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 0x3c00, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 0x3c00, s7
+; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s6, |v1.l|, 0.5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0x7fff, v3, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 0x3c00, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0x7fff, v2, s2
+; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, s3
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.h, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, v0.h, v2.l
+; GFX11-TRUE16-NEXT: buffer_store_b32 v1, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: round_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/true16-uniform-f16-phi-copysign.ll b/llvm/test/CodeGen/AMDGPU/true16-uniform-f16-phi-copysign.ll
new file mode 100644
index 00000000000000..e976d8fbc9347b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/true16-uniform-f16-phi-copysign.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck %s
+
+; An f16 value merged by a PHI across a wave-uniform branch is held in a 32-bit
+; register. The true16 fcopysign pattern used to feed it to a REG_SEQUENCE lo16
+; slot, naming a 16-bit slot for a 32-bit operand; DetectDeadLanes read that as
+; a lane-for-lane transfer and marked the value dead, so %big was emitted empty
+; and the stored result was undefined.
+;
+; %big now keeps its computation, and the uniform copysign reads its sign
+; operand straight out of an SGPR instead of copying it into a VGPR first.
+
+define amdgpu_kernel void @uniform_f16_phi_copysign(half %x, ptr addrspace(1) %out) {
+; CHECK-LABEL: uniform_f16_phi_copysign:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; CHECK-NEXT: s_mov_b64 s[64:65], 0
+; CHECK-NEXT: v_nop
+; CHECK-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: s_cmp_nle_f16 s0, 0x3800
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
+; CHECK-NEXT: ; %bb.1: ; %small
+; CHECK-NEXT: s_mul_f16 s1, s0, s0
+; CHECK-NEXT: s_mov_b32 s2, 0
+; CHECK-NEXT: s_branch .LBB0_3
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: s_mov_b32 s2, -1
+; CHECK-NEXT: ; implicit-def: $sgpr1
+; CHECK-NEXT: .LBB0_3: ; %Flow
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT: s_and_b32 s2, s2, exec_lo
+; CHECK-NEXT: s_cselect_b32 s2, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s2, 1
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_5
+; CHECK-NEXT: ; %bb.4: ; %big
+; CHECK-NEXT: s_cvt_f32_f16 s1, s0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0.5
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; CHECK-NEXT: v_s_sqrt_f32 s1, s1
+; CHECK-NEXT: v_nop
+; CHECK-NEXT: v_fma_mixlo_f16 v0, s1, s0, v0 op_sel_hi:[0,1,0]
+; CHECK-NEXT: s_branch .LBB0_6
+; CHECK-NEXT: .LBB0_5:
+; CHECK-NEXT: v_mov_b32_e32 v0, s1
+; CHECK-NEXT: .LBB0_6: ; %exit
+; CHECK-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; CHECK-NEXT: v_bfi_b32 v0, 0x7fff, v0, s0
+; CHECK-NEXT: s_wait_kmcnt 0x0
+; CHECK-NEXT: global_store_b16 v1, v0, s[2:3]
+; CHECK-NEXT: s_endpgm
+entry:
+ %c = fcmp ugt half %x, 0xH3800
+ br i1 %c, label %big, label %small
+
+small:
+ %s = fmul half %x, %x
+ br label %exit
+
+big:
+ %e = fpext half %x to float
+ %q = call float @llvm.amdgcn.sqrt.f32(float %e)
+ %m = call float @llvm.fmuladd.f32(float %q, float %e, float 5.000000e-01)
+ %t = fptrunc float %m to half
+ br label %exit
+
+exit:
+ %p = phi half [ %s, %small ], [ %t, %big ]
+ %r = call half @llvm.copysign.f16(half %p, half %x)
+ store half %r, ptr addrspace(1) %out
+ ret void
+}
More information about the llvm-commits
mailing list