[llvm] [AMDGPU] Optimize SGPR splat reg sequences for packed 64-bit instructions (PR #217182)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 18:00:04 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Changpeng Fang (changpeng)
<details>
<summary>Changes</summary>
This work optimizes SGPR register sequences for packed 64-bit instructions by eliminating
redundant register copies when a scalar value is splatted across multiple lanes.
The optimization is implemented in SIFoldOperands::tryFoldSGPRSplatRegSequence, which:
1. Detects SGPR register sequences where all elements are identical
2. Verifies all uses are packed 64-bit instructions supporting single SGPR read
3. Replaces redundant elements with undef to allow copy elimination
This reduces SGPR register pressure and eliminates unnecessary copies in kernels using packed
operations with splat scalars.
---
Full diff: https://github.com/llvm/llvm-project/pull/217182.diff
5 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+77)
- (modified) llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fp64.ll (+8-14)
- (modified) llvm/test/CodeGen/AMDGPU/packed-u64.ll (-1)
- (modified) llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll (+2-5)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index efbe5cf281f0f..b5bad8be59447 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -262,6 +262,7 @@ class SIFoldOperandsImpl {
std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
bool tryFoldOMod(MachineInstr &MI);
+ bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
bool tryFoldRegSequence(MachineInstr &MI);
bool tryFoldPhiAGPR(MachineInstr &MI);
bool tryFoldLoad(MachineInstr &MI);
@@ -2473,10 +2474,86 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
return true;
}
+// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
+// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
+bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
+ assert(MI.isRegSequence());
+ Register Reg = MI.getOperand(0).getReg();
+
+ // Only optimize SGPR register sequences
+ if (!TRI->isSGPRClass(MRI->getRegClass(Reg)))
+ return false;
+
+ SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
+ if (!getRegSeqInit(Defs, Reg))
+ return false;
+
+ // Check if this is a splat pattern
+ if (Defs.size() <= 1)
+ return false;
+
+ const auto &[FirstOp, _] = Defs.front();
+ if (!FirstOp->isReg())
+ return false;
+
+ Register FirstReg = FirstOp->getReg();
+ unsigned FirstSubReg = FirstOp->getSubReg();
+
+ if (!TRI->isSGPRClass(MRI->getRegClass(FirstReg)))
+ return false;
+
+ // Check remaining elements match first
+ if (!llvm::all_of(llvm::drop_begin(Defs), [&](const auto &Def) {
+ const auto &[Op, _] = Def;
+ return Op->isReg() && Op->getReg() == FirstReg &&
+ Op->getSubReg() == FirstSubReg;
+ }))
+ return false;
+
+ // Check if all uses are isSingleSGPRReadInst
+ for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
+ if (!AMDGPU::isPackedSingleSGPR64BitInst(UseMI.getOpcode()))
+ return false;
+ }
+
+ // Create new reg sequence with <s, undef, undef, ...>
+ auto NewDst = MRI->createVirtualRegister(MRI->getRegClass(Reg));
+ auto RS = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::REG_SEQUENCE), NewDst);
+
+ // Add the first operand
+ FirstOp->setIsKill(false);
+ RS.add(*FirstOp);
+ RS.addImm(Defs[0].second);
+
+ // Add undef for remaining lanes
+ // Create an undef virtual register for the same register class
+ Register UndefReg = MRI->createVirtualRegister(
+ MRI->getRegClass(FirstOp->getReg()));
+ for (unsigned i = 1; i < Defs.size(); ++i) {
+ RS.addReg(UndefReg, RegState::Undef);
+ RS.addImm(Defs[i].second);
+ }
+
+ // Replace all uses
+ MRI->replaceRegWith(Reg, NewDst);
+
+ LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI
+ << " into " << *RS);
+
+ MI.eraseFromParent();
+ return true;
+}
+
// Try to fold a reg_sequence with vgpr output and agpr inputs into an
// instruction which can take an agpr. So far that means a store.
bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
assert(MI.isRegSequence());
+
+ // Try to optimize SGPR splat sequences first
+ if (tryFoldSGPRSplatRegSequence(MI))
+ return true;
+
auto Reg = MI.getOperand(0).getReg();
if (!ST->hasGFX90AInsts() || !TRI->isVGPR(*MRI, Reg) ||
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
index 97a4969dd13ef..5c1a608841923 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
@@ -322,7 +322,7 @@ body: |
; GFX1251-NEXT: {{ $}}
; GFX1251-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1251-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GFX1251-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[DEF]], %subreg.sub0_sub1, killed [[DEF]], %subreg.sub2_sub3
+ ; GFX1251-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[DEF]], %subreg.sub0_sub1, undef %5:sreg_64, %subreg.sub2_sub3
; GFX1251-NEXT: [[V_PK_ADD_NC_U64_:%[0-9]+]]:vreg_128_align2 = V_PK_ADD_NC_U64 8, killed [[COPY]], 0, killed [[REG_SEQUENCE]], 0, 0, 0, implicit $exec
; GFX1251-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_NC_U64_]]
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index e0374fded2c88..50563ada51924 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -482,7 +482,6 @@ define amdgpu_kernel void @fadd_v2_v_v_sgpr_splat(ptr addrspace(1) %a, double %v
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -684,7 +683,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -735,7 +733,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -786,7 +783,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -1484,7 +1480,6 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -2077,7 +2072,6 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], s[4:7], s[4:7] neg_lo:[0,1,1] neg_hi:[0,1,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -2750,18 +2744,18 @@ define amdgpu_kernel void @fma_v2_s_imm_imm(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], 0x4069000000000000
+; GFX1251-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], 0x4069000000000000
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[6:7], 0x0
+; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], s[0:1]
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[2:5], 0x40590000, s[0:3], v[2:5]
-; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[6:7] scale_offset
+; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[2:3] scale_offset
; GFX1251-SDAG-NEXT: s_endpgm
;
; GFX1251-GISEL-LABEL: fma_v2_s_imm_imm:
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index ee86ba72093f7..71bc83860ffb2 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -482,7 +482,6 @@ define amdgpu_kernel void @add_v2_v_v_sgpr_splat(ptr addrspace(1) %a, i64 %v) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index 427465f5a5b35..8af990338f3d3 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -261,10 +261,8 @@ define <2 x i64> @pk_lshl_add_u64_v1_sgpr_splat(<2 x i64> %v, i64 inreg %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_mov_b32 s2, 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
-; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[0:3], v[4:5], s[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %a, i32 0
@@ -298,8 +296,7 @@ define <2 x i64> @pk_lshl_add_u64_sgpr_splat_v1(i64 inreg %v, <2 x i64> %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], s[0:3], v[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %v, i32 0
``````````
</details>
https://github.com/llvm/llvm-project/pull/217182
More information about the llvm-commits
mailing list