[llvm] [AMDGPU] Legalize SGPR hi16 copies into S_PACK_HH_B32_B16 (PR #218371)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 03:49:10 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
SGPR hi16 is an artificial subregister with no register class and cannot be allocated, so a copy reading it must become a pack instead
---
Full diff: https://github.com/llvm/llvm-project/pull/218371.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp (+50)
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+2-1)
- (modified) llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll (+115)
- (modified) llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir (+30)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 3bbc62412de9b..bdf01a6da56ae 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -626,6 +626,53 @@ static bool hoistAndMergeSGPRInits(unsigned Reg,
return Changed;
}
+static bool tryLegalizeHi16SGPRCopy(MachineInstr &MI, const SIRegisterInfo *TRI,
+ const SIInstrInfo *TII,
+ const GCNSubtarget &ST,
+ const TargetRegisterClass *SrcRC,
+ const TargetRegisterClass *DstRC) {
+ MachineOperand &Src = MI.getOperand(1);
+ unsigned SubReg = Src.getSubReg();
+ if (SubReg == AMDGPU::NoSubRegister || !ST.hasScalarPackInsts() ||
+ MI.getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
+ !TRI->isSGPRClass(SrcRC) || !TRI->isSGPRClass(DstRC) ||
+ TRI->getRegSizeInBits(*DstRC) != 32 ||
+ TRI->getSubRegIdxSize(SubReg) != 16)
+ return false;
+
+ unsigned SubRegOffset = TRI->getSubRegIdxOffset(SubReg);
+ if (SubRegOffset % 32 != 16)
+ return false;
+
+ MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ bool IsKill = Src.isKill();
+ bool IsUndef = Src.isUndef();
+ // Copy out the 32-bit channel first: movePackToVALU drops subreg sources
+ // if the pack is later forced to VALU.
+ if (TRI->getRegSizeInBits(*SrcRC) != 32) {
+ unsigned Channel = SubRegOffset / 32;
+ Register Chan32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
+ Chan32)
+ .addReg(Src.getReg(),
+ getKillRegState(IsKill) | getUndefRegState(IsUndef),
+ TRI->getSubRegFromChannel(Channel));
+ Src.setReg(Chan32);
+ IsKill = true;
+ IsUndef = false;
+ }
+ // Cache the register: addReg below may reallocate MI's operands and
+ // invalidate Src.
+ Register SrcReg = Src.getReg();
+ MI.setDesc(TII->get(AMDGPU::S_PACK_HH_B32_B16));
+ Src.setSubReg(AMDGPU::NoSubRegister);
+ Src.setIsKill(false);
+ Src.setIsUndef(IsUndef);
+ MachineInstrBuilder(*MI.getMF(), MI)
+ .addReg(SrcReg, getKillRegState(IsKill) | getUndefRegState(IsUndef));
+ return true;
+}
+
bool SIFixSGPRCopies::run(MachineFunction &MF) {
// Only need to run this in SelectionDAG path.
if (MF.getProperties().hasSelected())
@@ -656,6 +703,9 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
const TargetRegisterClass *SrcRC, *DstRC;
std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
+ if (tryLegalizeHi16SGPRCopy(MI, TRI, TII, ST, SrcRC, DstRC))
+ continue;
+
if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
// Since VGPR to SGPR copies affect VGPR to SGPR copy
// score and, hence the lowering decision, let's try to get rid of
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 584efa001f2a5..6280bd19b6949 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2217,7 +2217,8 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
OpToFold.getSubReg()) {
if (DstRC == &AMDGPU::SReg_32RegClass &&
DstRC == MRI->getRegClass(OpToFold.getReg())) {
- assert(OpToFold.getSubReg() == AMDGPU::lo16);
+ if (OpToFold.getSubReg() != AMDGPU::lo16)
+ return false;
OpToFold.setSubReg(0);
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 9db4987fc8210..334b4e3b436dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -6767,6 +6767,121 @@ define half @v_copysign_f16_0_f32(float %sign) {
ret half %op
}
+; A VALU user of a uniform copysign result selects the true16 pattern
+; reading an SGPR hi16.
+define amdgpu_kernel void @s_copysign_f16_0_f32_valu_user(float %sign, i1 %cond) {
+; SI-LABEL: s_copysign_f16_0_f32_valu_user:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bitcmp1_b32 s1, 0
+; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
+; SI-NEXT: s_and_b64 vcc, exec, s[2:3]
+; SI-NEXT: s_cbranch_vccnz .LBB76_2
+; SI-NEXT: ; %bb.1: ; %if.end
+; SI-NEXT: s_and_b32 s0, s0, 0x80000000
+; SI-NEXT: s_lshr_b32 s0, s0, 16
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
+; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: .LBB76_2: ; %exit
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_copysign_f16_0_f32_valu_user:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_bitcmp1_b32 s1, 0
+; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
+; VI-NEXT: s_and_b64 vcc, exec, s[2:3]
+; VI-NEXT: s_cbranch_vccnz .LBB76_2
+; VI-NEXT: ; %bb.1: ; %if.end
+; VI-NEXT: s_lshr_b32 s0, s0, 16
+; VI-NEXT: s_and_b32 s0, 0x8000, s0
+; VI-NEXT: v_cvt_f32_f16_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: v_cvt_i32_f32_e32 v2, v0
+; VI-NEXT: v_mov_b32_e32 v0, 0
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: .LBB76_2: ; %exit
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_copysign_f16_0_f32_valu_user:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_bitcmp1_b32 s1, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GFX9-NEXT: s_cbranch_vccnz .LBB76_2
+; GFX9-NEXT: ; %bb.1: ; %if.end
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: s_and_b32 s0, 0x8000, s0
+; GFX9-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-NEXT: .LBB76_2: ; %exit
+; GFX9-NEXT: s_endpgm
+;
+; GFX11-TRUE16-LABEL: s_copysign_f16_0_f32_valu_user:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_bitcmp1_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 vcc_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB76_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %if.end
+; GFX11-TRUE16-NEXT: s_pack_hh_b32_b16 s0, s0, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: global_store_b32 v[0:1], v2, off
+; GFX11-TRUE16-NEXT: .LBB76_2: ; %exit
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: s_copysign_f16_0_f32_valu_user:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_bitcmp1_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 vcc_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB76_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %if.end
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-FAKE16-NEXT: s_and_b32 s0, 0x8000, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, s0
+; GFX11-FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11-FAKE16-NEXT: global_store_b32 v[0:1], v2, off
+; GFX11-FAKE16-NEXT: .LBB76_2: ; %exit
+; GFX11-FAKE16-NEXT: s_endpgm
+entry:
+ br i1 %cond, label %exit, label %if.end
+
+if.end:
+ %result = call float @llvm.copysign.f32(float 0.0, float %sign)
+ %trunc = fptrunc float %result to half
+ %cvt = fptosi half %trunc to i32
+ store i32 %cvt, ptr addrspace(1) null, align 4
+ br label %exit
+
+exit:
+ ret void
+}
+
define amdgpu_ps i32 @s_copysign_f16_0_f64(double inreg %sign) {
; SI-LABEL: s_copysign_f16_0_f64:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
index 6c96be0860b43..d6d28e7df9af4 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
@@ -464,3 +464,33 @@ body: |
%2:sreg_32 = COPY %1.hi16
%3:vgpr_16 = V_OR_B16_t16_e64 0, %0.lo16, 0, killed %2, 0, implicit $exec
...
+
+# SGPR hi16 is an artificial register that cannot be allocated; must become
+# S_PACK_HH.
+
+---
+name: sgpr32_hi16_to_sgpr32
+body: |
+ bb.0:
+ ; GCN-LABEL: name: sgpr32_hi16_to_sgpr32
+ ; GCN: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; GCN-NEXT: [[S_PACK_HH_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_HH_B32_B16 [[DEF]], [[DEF]]
+ ; GCN-NEXT: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_PACK_HH_B32_B16_]], [[S_PACK_HH_B32_B16_]], implicit-def dead $scc
+ %0:sreg_32 = IMPLICIT_DEF
+ %1:sreg_32 = COPY %0.hi16
+ %2:sreg_32 = S_PACK_LL_B32_B16 %1:sreg_32, %1:sreg_32, implicit-def dead $scc
+...
+
+---
+name: sgpr64_sub1_hi16_to_sgpr32
+body: |
+ bb.0:
+ ; GCN-LABEL: name: sgpr64_sub1_hi16_to_sgpr32
+ ; GCN: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1
+ ; GCN-NEXT: [[S_PACK_HH_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_HH_B32_B16 [[COPY]], killed [[COPY]]
+ ; GCN-NEXT: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[S_PACK_HH_B32_B16_]], [[S_PACK_HH_B32_B16_]], implicit-def dead $scc
+ %0:sreg_64 = IMPLICIT_DEF
+ %1:sreg_32 = COPY %0.sub1_hi16
+ %2:sreg_32 = S_PACK_LL_B32_B16 %1:sreg_32, %1:sreg_32, implicit-def dead $scc
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/218371
More information about the llvm-commits
mailing list