[llvm] [WIP][AMDGPU] Improve the handling of `inreg` arguments (PR #133614)
Shilei Tian via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 22 10:26:43 PST 2026
================
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-inreg-vgpr-lane-packing < %s | FileCheck %s
+
+; Test that overflow inreg arguments are packed into VGPR lanes using
+; writelane/readlane instead of consuming individual VGPRs.
+;
+; Each test has a callee + caller pair so we can verify:
+; - Callee: overflow inreg args are extracted via v_readlane_b32 into SGPRs
+; and used in s_ (SALU) instructions
+; - Caller: overflow inreg args are packed via v_writelane_b32 into the same
+; VGPR and lane that the callee reads from
+
+; --- Test 1: SGPR arg + first overflow arg ---
+; Callee reads a0 from SGPR and a14 from VGPR lane 0, adds with s_add.
+; Caller writes 100 to the SGPR for a0 and 200 to VGPR lane 0 for a14.
+define i32 @callee_add_sgpr_and_overflow(
+; CHECK-LABEL: callee_add_sgpr_and_overflow:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_readlane_b32 s4, v0, 0
+; CHECK-NEXT: s_add_i32 s16, s16, s4
+; CHECK-NEXT: v_mov_b32_e32 v0, s16
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ i32 inreg %a0, i32 inreg %a1, i32 inreg %a2, i32 inreg %a3,
+ i32 inreg %a4, i32 inreg %a5, i32 inreg %a6, i32 inreg %a7,
+ i32 inreg %a8, i32 inreg %a9, i32 inreg %a10, i32 inreg %a11,
+ i32 inreg %a12, i32 inreg %a13, i32 inreg %a14, i32 inreg %a15,
+ i32 inreg %a16, i32 inreg %a17, i32 inreg %a18, i32 inreg %a19,
+ i32 inreg %a20, i32 inreg %a21, i32 inreg %a22, i32 inreg %a23,
+ i32 inreg %a24, i32 inreg %a25, i32 inreg %a26, i32 inreg %a27,
+ i32 inreg %a28, i32 inreg %a29, i32 inreg %a30, i32 inreg %a31) {
+ %sum = add i32 %a0, %a14
+ ret i32 %sum
+}
+
+define i32 @caller_add_sgpr_and_overflow() {
+; CHECK-LABEL: caller_add_sgpr_and_overflow:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s42, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1
+; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[16:17]
+; CHECK-NEXT: s_movk_i32 s16, 0xc8
+; CHECK-NEXT: v_writelane_b32 v0, s16, 0
+; CHECK-NEXT: v_writelane_b32 v0, 15, 1
+; CHECK-NEXT: v_writelane_b32 v0, 16, 2
+; CHECK-NEXT: v_writelane_b32 v0, 17, 3
+; CHECK-NEXT: v_writelane_b32 v0, 18, 4
+; CHECK-NEXT: v_writelane_b32 v0, 19, 5
+; CHECK-NEXT: v_writelane_b32 v0, 20, 6
+; CHECK-NEXT: v_writelane_b32 v0, 21, 7
+; CHECK-NEXT: v_writelane_b32 v0, 22, 8
+; CHECK-NEXT: v_writelane_b32 v0, 23, 9
+; CHECK-NEXT: v_writelane_b32 v0, 24, 10
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: v_writelane_b32 v0, 25, 11
+; CHECK-NEXT: v_writelane_b32 v0, 26, 12
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, callee_add_sgpr_and_overflow at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, callee_add_sgpr_and_overflow at gotpcrel32@hi+12
+; CHECK-NEXT: v_writelane_b32 v0, 27, 13
+; CHECK-NEXT: s_load_dwordx2 s[40:41], s[16:17], 0x0
+; CHECK-NEXT: v_writelane_b32 v0, 28, 14
+; CHECK-NEXT: v_writelane_b32 v0, 29, 15
+; CHECK-NEXT: v_writelane_b32 v0, 30, 16
+; CHECK-NEXT: v_writelane_b32 v1, s30, 0
+; CHECK-NEXT: v_writelane_b32 v0, 31, 17
+; CHECK-NEXT: s_movk_i32 s16, 0x64
+; CHECK-NEXT: s_mov_b32 s17, 1
+; CHECK-NEXT: s_mov_b32 s18, 2
+; CHECK-NEXT: s_mov_b32 s19, 3
+; CHECK-NEXT: s_mov_b32 s20, 4
+; CHECK-NEXT: s_mov_b32 s21, 5
+; CHECK-NEXT: s_mov_b32 s22, 6
+; CHECK-NEXT: s_mov_b32 s23, 7
+; CHECK-NEXT: s_mov_b32 s24, 8
+; CHECK-NEXT: s_mov_b32 s25, 9
+; CHECK-NEXT: s_mov_b32 s26, 10
+; CHECK-NEXT: s_mov_b32 s27, 11
+; CHECK-NEXT: s_mov_b32 s28, 12
+; CHECK-NEXT: s_mov_b32 s29, 13
+; CHECK-NEXT: v_writelane_b32 v1, s31, 1
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[40:41]
+; CHECK-NEXT: v_readlane_b32 s31, v1, 1
+; CHECK-NEXT: v_readlane_b32 s30, v1, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[4:5]
+; CHECK-NEXT: s_mov_b32 s33, s42
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %r = call i32 @callee_add_sgpr_and_overflow(
+ i32 inreg 100, i32 inreg 1, i32 inreg 2, i32 inreg 3,
+ i32 inreg 4, i32 inreg 5, i32 inreg 6, i32 inreg 7,
+ i32 inreg 8, i32 inreg 9, i32 inreg 10, i32 inreg 11,
+ i32 inreg 12, i32 inreg 13, i32 inreg 200, i32 inreg 15,
+ i32 inreg 16, i32 inreg 17, i32 inreg 18, i32 inreg 19,
+ i32 inreg 20, i32 inreg 21, i32 inreg 22, i32 inreg 23,
+ i32 inreg 24, i32 inreg 25, i32 inreg 26, i32 inreg 27,
+ i32 inreg 28, i32 inreg 29, i32 inreg 30, i32 inreg 31)
+ ret i32 %r
+}
+
+; --- Test 2: two overflow args from different VGPR lanes ---
+; Callee reads a30 and a31 from high lanes of the same VGPR, adds with s_add.
+; Caller writes 42 and 58 to those lanes via writelane.
+define i32 @callee_add_two_overflow(
+; CHECK-LABEL: callee_add_two_overflow:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_readlane_b32 s4, v0, 17
+; CHECK-NEXT: v_readlane_b32 s5, v0, 16
+; CHECK-NEXT: s_add_i32 s5, s5, s4
+; CHECK-NEXT: v_mov_b32_e32 v0, s5
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ i32 inreg %a0, i32 inreg %a1, i32 inreg %a2, i32 inreg %a3,
+ i32 inreg %a4, i32 inreg %a5, i32 inreg %a6, i32 inreg %a7,
+ i32 inreg %a8, i32 inreg %a9, i32 inreg %a10, i32 inreg %a11,
+ i32 inreg %a12, i32 inreg %a13, i32 inreg %a14, i32 inreg %a15,
+ i32 inreg %a16, i32 inreg %a17, i32 inreg %a18, i32 inreg %a19,
+ i32 inreg %a20, i32 inreg %a21, i32 inreg %a22, i32 inreg %a23,
+ i32 inreg %a24, i32 inreg %a25, i32 inreg %a26, i32 inreg %a27,
+ i32 inreg %a28, i32 inreg %a29, i32 inreg %a30, i32 inreg %a31) {
+ %sum = add i32 %a30, %a31
+ ret i32 %sum
+}
+
+define i32 @caller_add_two_overflow() {
+; CHECK-LABEL: caller_add_two_overflow:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s42, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1
+; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[16:17]
+; CHECK-NEXT: v_writelane_b32 v0, 14, 0
+; CHECK-NEXT: v_writelane_b32 v0, 15, 1
+; CHECK-NEXT: v_writelane_b32 v0, 16, 2
+; CHECK-NEXT: v_writelane_b32 v0, 17, 3
+; CHECK-NEXT: v_writelane_b32 v0, 18, 4
+; CHECK-NEXT: v_writelane_b32 v0, 19, 5
+; CHECK-NEXT: v_writelane_b32 v0, 20, 6
+; CHECK-NEXT: v_writelane_b32 v0, 21, 7
+; CHECK-NEXT: v_writelane_b32 v0, 22, 8
+; CHECK-NEXT: v_writelane_b32 v0, 23, 9
+; CHECK-NEXT: v_writelane_b32 v0, 24, 10
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: v_writelane_b32 v0, 25, 11
+; CHECK-NEXT: v_writelane_b32 v0, 26, 12
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, callee_add_two_overflow at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, callee_add_two_overflow at gotpcrel32@hi+12
+; CHECK-NEXT: v_writelane_b32 v0, 27, 13
+; CHECK-NEXT: s_load_dwordx2 s[40:41], s[16:17], 0x0
+; CHECK-NEXT: v_writelane_b32 v0, 28, 14
+; CHECK-NEXT: v_writelane_b32 v0, 29, 15
+; CHECK-NEXT: v_writelane_b32 v0, 42, 16
+; CHECK-NEXT: v_writelane_b32 v1, s30, 0
+; CHECK-NEXT: v_writelane_b32 v0, 58, 17
+; CHECK-NEXT: s_mov_b32 s16, 0
+; CHECK-NEXT: s_mov_b32 s17, 1
+; CHECK-NEXT: s_mov_b32 s18, 2
+; CHECK-NEXT: s_mov_b32 s19, 3
+; CHECK-NEXT: s_mov_b32 s20, 4
+; CHECK-NEXT: s_mov_b32 s21, 5
+; CHECK-NEXT: s_mov_b32 s22, 6
+; CHECK-NEXT: s_mov_b32 s23, 7
+; CHECK-NEXT: s_mov_b32 s24, 8
+; CHECK-NEXT: s_mov_b32 s25, 9
+; CHECK-NEXT: s_mov_b32 s26, 10
+; CHECK-NEXT: s_mov_b32 s27, 11
+; CHECK-NEXT: s_mov_b32 s28, 12
+; CHECK-NEXT: s_mov_b32 s29, 13
+; CHECK-NEXT: v_writelane_b32 v1, s31, 1
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[40:41]
+; CHECK-NEXT: v_readlane_b32 s31, v1, 1
+; CHECK-NEXT: v_readlane_b32 s30, v1, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[4:5]
+; CHECK-NEXT: s_mov_b32 s33, s42
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %r = call i32 @callee_add_two_overflow(
+ i32 inreg 0, i32 inreg 1, i32 inreg 2, i32 inreg 3,
+ i32 inreg 4, i32 inreg 5, i32 inreg 6, i32 inreg 7,
+ i32 inreg 8, i32 inreg 9, i32 inreg 10, i32 inreg 11,
+ i32 inreg 12, i32 inreg 13, i32 inreg 14, i32 inreg 15,
+ i32 inreg 16, i32 inreg 17, i32 inreg 18, i32 inreg 19,
+ i32 inreg 20, i32 inreg 21, i32 inreg 22, i32 inreg 23,
+ i32 inreg 24, i32 inreg 25, i32 inreg 26, i32 inreg 27,
+ i32 inreg 28, i32 inreg 29, i32 inreg 42, i32 inreg 58)
+ ret i32 %r
+}
+
+; --- Test 3: mixed inreg and non-inreg ---
+; Callee: s0 (SGPR) + s16 (overflow inreg from VGPR lane) -> s_add,
+; then + v0 (normal VGPR arg) -> v_add.
+; Caller writes 10 to SGPR for s0, 20 to VGPR lane for s16, and passes
+; a VGPR value for v0.
+define i32 @callee_mixed(
+; CHECK-LABEL: callee_mixed:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_readlane_b32 s4, v0, 2
+; CHECK-NEXT: s_add_i32 s16, s16, s4
+; CHECK-NEXT: v_add_u32_e32 v0, s16, v1
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ i32 inreg %s0, i32 inreg %s1, i32 inreg %s2, i32 inreg %s3,
+ i32 inreg %s4, i32 inreg %s5, i32 inreg %s6, i32 inreg %s7,
+ i32 inreg %s8, i32 inreg %s9, i32 inreg %s10, i32 inreg %s11,
+ i32 inreg %s12, i32 inreg %s13, i32 inreg %s14, i32 inreg %s15,
+ i32 %v0,
+ i32 inreg %s16, i32 inreg %s17) {
+ %s_sum = add i32 %s0, %s16
+ %result = add i32 %s_sum, %v0
+ ret i32 %result
+}
+
+define i32 @caller_mixed(i32 %vgpr_val) {
+; CHECK-LABEL: caller_mixed:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s42, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[16:17]
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, callee_mixed at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, callee_mixed at gotpcrel32@hi+12
+; CHECK-NEXT: v_writelane_b32 v2, 14, 0
+; CHECK-NEXT: s_load_dwordx2 s[40:41], s[16:17], 0x0
+; CHECK-NEXT: v_writelane_b32 v2, 15, 1
+; CHECK-NEXT: v_writelane_b32 v2, 20, 2
+; CHECK-NEXT: v_writelane_b32 v2, 21, 3
+; CHECK-NEXT: v_writelane_b32 v3, s30, 0
+; CHECK-NEXT: s_mov_b32 s16, 10
+; CHECK-NEXT: s_mov_b32 s17, 1
+; CHECK-NEXT: s_mov_b32 s18, 2
+; CHECK-NEXT: s_mov_b32 s19, 3
+; CHECK-NEXT: s_mov_b32 s20, 4
+; CHECK-NEXT: s_mov_b32 s21, 5
+; CHECK-NEXT: s_mov_b32 s22, 6
+; CHECK-NEXT: s_mov_b32 s23, 7
+; CHECK-NEXT: s_mov_b32 s24, 8
+; CHECK-NEXT: s_mov_b32 s25, 9
+; CHECK-NEXT: s_mov_b32 s26, 10
+; CHECK-NEXT: s_mov_b32 s27, 11
+; CHECK-NEXT: s_mov_b32 s28, 12
+; CHECK-NEXT: s_mov_b32 s29, 13
+; CHECK-NEXT: v_mov_b32_e32 v1, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, v2
----------------
shiltian wrote:
Here it shows the issue when coalescing fails, we might run into issue, because at call site, the writelane writes to v2 followed by a copy from v2 to v0, and then in the callee, the readlane directly reads from v0. If the caller is in a divergent branch, then this would not correctly.
However, the challenging problem is, how can we issue a WWM_COPY at ISel stage? Maybe we need to add a pseudo node and then change it to something else later? The current WWM_COPY rewriting pass seems like only tailored for VGPR spilling.
https://github.com/llvm/llvm-project/pull/133614
More information about the llvm-commits
mailing list