[llvm] [AMDGPU] Fold sreg32 operand across 16bit subcopy (PR #215375)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 13:09:48 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Guo Chen (broxigarchen)
<details>
<summary>Changes</summary>
One step peephole in FoldOperand that fold sreg32 across 16bit subcopy into VALU16 operands
---
Full diff: https://github.com/llvm/llvm-project/pull/215375.diff
6 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+39)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll (+26-58)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll (+4-11)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir (+1-2)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll (+1-3)
- (modified) llvm/test/CodeGen/AMDGPU/true16-fold.mir (+62)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 04a9ed487d655..19ad398dab427 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1508,6 +1508,45 @@ bool SIFoldOperandsImpl::foldOperand(
return true;
}
+ // Look through Lo16 Copy
+ // OpToFold: %0
+ // %1:vgpr32 = copy %0:sreg32/sregxx:sub_x
+ // %2:vgpr16 = copy %1.lo16:vgpr32 (UseMI)
+ // VALU16 %2:vgpr16 ...
+ // =>
+ // VALU16 %0:sreg32/sregxx:sub_x...
+ // Another hack to allow 32-bit SGPRs to be folded into True16 instructions
+ // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
+ // VS_16RegClass
+ if (UseMI->isCopy() && OpToFold.isReg() &&
+ UseMI->getOperand(0).getReg().isVirtual() &&
+ TRI->isSGPRReg(*MRI, OpToFold.getReg()) &&
+ TII->getOpSize(*UseMI, 0) == 2 &&
+ UseMI->getOperand(1).getSubReg() == AMDGPU::lo16 &&
+ OpToFold.DefMI->implicit_operands().empty()) {
+
+ // Append Users of the UseMI instead
+ SmallVector<MachineOperand *, 4> UsesToProcess(llvm::make_pointer_range(
+ MRI->use_nodbg_operands(UseMI->getOperand(0).getReg())));
+ for (auto *U : UsesToProcess) {
+ MachineInstr *NextMI = U->getParent();
+
+ const MCInstrDesc &UseDesc = NextMI->getDesc();
+ if (UseDesc.isVariadic() || U->isImplicit() ||
+ UseDesc.operands()[NextMI->getOperandNo(U)].RegClass == -1)
+ continue;
+
+ FoldableDef NextOpToFold(*OpToFold.OpToFold, OpToFold.DefRC,
+ U->getSubReg());
+ LLVM_DEBUG(dbgs() << "Folding " << *NextOpToFold.OpToFold
+ << "\n through" << *UseMI << " into " << *NextMI);
+
+ Changed |= tryAddToFoldList(FoldList, NextMI, NextMI->getOperandNo(U),
+ NextOpToFold);
+ }
+ return Changed;
+ }
+
unsigned UseOpc = UseMI->getOpcode();
if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
(UseOpc == AMDGPU::V_READLANE_B32 &&
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 6844b698473c7..14294565c0f63 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -3028,28 +3028,16 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_fshl_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_fshl_i32:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_fshl_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
ret i32 %result
}
@@ -3289,24 +3277,14 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshl_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s1, s1
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshl_i32_svs:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshl_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
@@ -3351,24 +3329,14 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshl_i32_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshl_i32_vss:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshl_i32_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 11b62d825c965..ee863cc06eb6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -3257,17 +3257,10 @@ define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshr_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshr_i32_svs:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshr_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
index b0587821eb116..df9d756bdd0bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
@@ -136,8 +136,7 @@ body: |
; GFX11-FOLD-TRUE16-NEXT: {{ $}}
; GFX11-FOLD-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-FOLD-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX11-FOLD-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX11-FOLD-TRUE16-NEXT: [[V_CMP_CLASS_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_CLASS_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY2]].lo16, 0, implicit $exec
+ ; GFX11-FOLD-TRUE16-NEXT: [[V_CMP_CLASS_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_CLASS_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY1]], 0, implicit $exec
; GFX11-FOLD-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_CLASS_F16_t16_e64_]]
;
; GFX11-FOLD-FAKE16-LABEL: name: class_f16_vcc_vs
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
index de05fe56b8428..db8687edb2642 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
@@ -294,9 +294,7 @@ define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspac
; GFX11-TRUE16-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x3c
; GFX11-TRUE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX11-TRUE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-GISEL-NEXT: v_alignbyte_b32 v0, v1, v0, v2.l
+; GFX11-TRUE16-GISEL-NEXT: v_alignbyte_b32 v0, v1, v0, s2
; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-TRUE16-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/true16-fold.mir b/llvm/test/CodeGen/AMDGPU/true16-fold.mir
index 29be6c55e6f30..5a12046abc61c 100644
--- a/llvm/test/CodeGen/AMDGPU/true16-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/true16-fold.mir
@@ -265,3 +265,65 @@ body: |
$vgpr0 = COPY %5
SI_RETURN implicit $vgpr0
...
+
+---
+name: fold_sreg32_cross_lo16_copy_1
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_lo16_copy_1
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[COPY]], 0, [[COPY]], -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY %0:sreg_32
+ %2:vgpr_16 = COPY %1.lo16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: fold_sreg32_cross_lo16_copy_2
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_lo16_copy_2
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[DEF]].sub1, 0, [[DEF]].sub1, -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_64 = IMPLICIT_DEF
+ %1:vgpr_32 = COPY %0.sub1:sreg_64
+ %2:vgpr_16 = COPY %1.lo16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
+# Should not fold
+---
+name: fold_sreg32_cross_hi16
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_hi16
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[COPY1]].hi16, 0, [[COPY1]].hi16, -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY %0:sreg_32
+ %2:vgpr_16 = COPY %1.hi16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/215375
More information about the llvm-commits
mailing list