[llvm] [AMDGPU] Optimize SGPR splat reg sequences for packed 64-bit instructions (PR #217182)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 17:59:22 PDT 2026
https://github.com/changpeng created https://github.com/llvm/llvm-project/pull/217182
This work optimizes SGPR register sequences for packed 64-bit instructions by eliminating
redundant register copies when a scalar value is splatted across multiple lanes.
The optimization is implemented in SIFoldOperands::tryFoldSGPRSplatRegSequence, which:
1. Detects SGPR register sequences where all elements are identical
2. Verifies all uses are packed 64-bit instructions supporting single SGPR read
3. Replaces redundant elements with undef to allow copy elimination
This reduces SGPR register pressure and eliminates unnecessary copies in kernels using packed
operations with splat scalars.
>From 247c66e090f3c4ea5835ce6068d1455f753d110f Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Tue, 18 Aug 2026 16:24:37 -0700
Subject: [PATCH] [AMDGPU] Optimize SGPR splat reg sequences for packed 64-bit
instructions
This work optimizes SGPR register sequences for packed 64-bit instructions
by eliminating redundant register copies when a scalar value is splatted
across multiple lanes.
The optimization is implemented in SIFoldOperands::tryFoldSGPRSplatRegSequence,
which:
1. Detects SGPR register sequences where all elements are identical
2. Verifies all uses are packed 64-bit instructions supporting single SGPR read
3. Replaces redundant elements with undef to allow copy elimination
This reduces SGPR register pressure and eliminates unnecessary copies in
kernels using packed operations with splat scalar operands.
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 77 +++++++++++++++++++++
llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir | 2 +-
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 22 +++---
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 1 -
llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll | 7 +-
5 files changed, 88 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index efbe5cf281f0f..b5bad8be59447 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -262,6 +262,7 @@ class SIFoldOperandsImpl {
std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
bool tryFoldOMod(MachineInstr &MI);
+ bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
bool tryFoldRegSequence(MachineInstr &MI);
bool tryFoldPhiAGPR(MachineInstr &MI);
bool tryFoldLoad(MachineInstr &MI);
@@ -2473,10 +2474,86 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
return true;
}
+// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
+// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
+bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
+ assert(MI.isRegSequence());
+ Register Reg = MI.getOperand(0).getReg();
+
+ // Only optimize SGPR register sequences
+ if (!TRI->isSGPRClass(MRI->getRegClass(Reg)))
+ return false;
+
+ SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
+ if (!getRegSeqInit(Defs, Reg))
+ return false;
+
+ // Check if this is a splat pattern
+ if (Defs.size() <= 1)
+ return false;
+
+ const auto &[FirstOp, _] = Defs.front();
+ if (!FirstOp->isReg())
+ return false;
+
+ Register FirstReg = FirstOp->getReg();
+ unsigned FirstSubReg = FirstOp->getSubReg();
+
+ if (!TRI->isSGPRClass(MRI->getRegClass(FirstReg)))
+ return false;
+
+ // Check remaining elements match first
+ if (!llvm::all_of(llvm::drop_begin(Defs), [&](const auto &Def) {
+ const auto &[Op, _] = Def;
+ return Op->isReg() && Op->getReg() == FirstReg &&
+ Op->getSubReg() == FirstSubReg;
+ }))
+ return false;
+
+ // Check if all uses are isSingleSGPRReadInst
+ for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
+ if (!AMDGPU::isPackedSingleSGPR64BitInst(UseMI.getOpcode()))
+ return false;
+ }
+
+ // Create new reg sequence with <s, undef, undef, ...>
+ auto NewDst = MRI->createVirtualRegister(MRI->getRegClass(Reg));
+ auto RS = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
+ TII->get(AMDGPU::REG_SEQUENCE), NewDst);
+
+ // Add the first operand
+ FirstOp->setIsKill(false);
+ RS.add(*FirstOp);
+ RS.addImm(Defs[0].second);
+
+ // Add undef for remaining lanes
+ // Create an undef virtual register for the same register class
+ Register UndefReg = MRI->createVirtualRegister(
+ MRI->getRegClass(FirstOp->getReg()));
+ for (unsigned i = 1; i < Defs.size(); ++i) {
+ RS.addReg(UndefReg, RegState::Undef);
+ RS.addImm(Defs[i].second);
+ }
+
+ // Replace all uses
+ MRI->replaceRegWith(Reg, NewDst);
+
+ LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI
+ << " into " << *RS);
+
+ MI.eraseFromParent();
+ return true;
+}
+
// Try to fold a reg_sequence with vgpr output and agpr inputs into an
// instruction which can take an agpr. So far that means a store.
bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
assert(MI.isRegSequence());
+
+ // Try to optimize SGPR splat sequences first
+ if (tryFoldSGPRSplatRegSequence(MI))
+ return true;
+
auto Reg = MI.getOperand(0).getReg();
if (!ST->hasGFX90AInsts() || !TRI->isVGPR(*MRI, Reg) ||
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
index 97a4969dd13ef..5c1a608841923 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-pk64.mir
@@ -322,7 +322,7 @@ body: |
; GFX1251-NEXT: {{ $}}
; GFX1251-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1251-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GFX1251-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[DEF]], %subreg.sub0_sub1, killed [[DEF]], %subreg.sub2_sub3
+ ; GFX1251-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[DEF]], %subreg.sub0_sub1, undef %5:sreg_64, %subreg.sub2_sub3
; GFX1251-NEXT: [[V_PK_ADD_NC_U64_:%[0-9]+]]:vreg_128_align2 = V_PK_ADD_NC_U64 8, killed [[COPY]], 0, killed [[REG_SEQUENCE]], 0, 0, 0, implicit $exec
; GFX1251-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_NC_U64_]]
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index e0374fded2c88..50563ada51924 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -482,7 +482,6 @@ define amdgpu_kernel void @fadd_v2_v_v_sgpr_splat(ptr addrspace(1) %a, double %v
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -684,7 +683,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -735,7 +733,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -786,7 +783,6 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], s[4:7] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -1484,7 +1480,6 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[0:3], s[4:7] neg_lo:[0,1] neg_hi:[0,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -2077,7 +2072,6 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], s[4:7], s[4:7] neg_lo:[0,1,1] neg_hi:[0,1,1]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
@@ -2750,18 +2744,18 @@ define amdgpu_kernel void @fma_v2_s_imm_imm(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1251-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
-; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], 0x4069000000000000
+; GFX1251-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], 0x4069000000000000
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[6:7], 0x0
+; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: s_mov_b64 s[2:3], s[0:1]
-; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[2:5], 0x40590000, s[0:3], v[2:5]
-; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[6:7] scale_offset
+; GFX1251-SDAG-NEXT: global_store_b128 v0, v[2:5], s[2:3] scale_offset
; GFX1251-SDAG-NEXT: s_endpgm
;
; GFX1251-GISEL-LABEL: fma_v2_s_imm_imm:
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index ee86ba72093f7..71bc83860ffb2 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -482,7 +482,6 @@ define amdgpu_kernel void @add_v2_v_v_sgpr_splat(ptr addrspace(1) %a, i64 %v) {
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1251-SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], s[4:7]
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1] scale_offset
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index 427465f5a5b35..8af990338f3d3 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -261,10 +261,8 @@ define <2 x i64> @pk_lshl_add_u64_v1_sgpr_splat(<2 x i64> %v, i64 inreg %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_mov_b32 s2, 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
-; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[0:3], v[4:5], s[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %a, i32 0
@@ -298,8 +296,7 @@ define <2 x i64> @pk_lshl_add_u64_sgpr_splat_v1(i64 inreg %v, <2 x i64> %a) {
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1251-NEXT: s_wait_kmcnt 0x0
; GFX1251-NEXT: s_lshl_b64 s[0:1], s[0:1], 1
-; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1251-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], s[0:3], v[0:3]
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%s1 = insertelement <2 x i64> poison, i64 %v, i32 0
More information about the llvm-commits
mailing list