[llvm] [AMDGPU] Fix undivergent parameter pass using vgpr (PR #178198)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 27 05:13:11 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: None (Shoreshen)
<details>
<summary>Changes</summary>
Fix inreg argument, which is un-divergent, but using VGPR due to run out of SGPR.
---
Full diff: https://github.com/llvm/llvm-project/pull/178198.diff
7 Files Affected:
- (modified) llvm/include/llvm/CodeGen/TargetCallingConv.h (+9-5)
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+6-4)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (+2)
- (modified) llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp (+3)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+5-2)
- (removed) llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll (-27)
- (added) llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill.ll (+156)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index f197c7f1645ec..afc8b6d1af84c 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -30,6 +30,7 @@ namespace ISD {
unsigned IsSExt : 1; ///< Sign extended
unsigned IsNoExt : 1; ///< No extension
unsigned IsInReg : 1; ///< Passed in register
+ unsigned IsArgInReg : 1; ///< Is caller function's 'inreg' argument
unsigned IsSRet : 1; ///< Hidden struct-ret ptr
unsigned IsByVal : 1; ///< Struct passed by value
unsigned IsByRef : 1; ///< Passed in memory
@@ -63,11 +64,11 @@ namespace ISD {
public:
ArgFlagsTy()
- : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsSRet(0), IsByVal(0),
- IsByRef(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0),
- IsPreallocated(0), IsSplitEnd(0), IsSwiftSelf(0), IsSwiftAsync(0),
- IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
- IsSecArgPass(0), MemAlign(0), OrigAlign(0),
+ : IsZExt(0), IsSExt(0), IsNoExt(0), IsInReg(0), IsArgInReg(0),
+ IsSRet(0), IsByVal(0), IsByRef(0), IsNest(0), IsReturned(0),
+ IsSplit(0), IsInAlloca(0), IsPreallocated(0), IsSplitEnd(0),
+ IsSwiftSelf(0), IsSwiftAsync(0), IsSwiftError(0), IsCFGuardTarget(0),
+ IsHva(0), IsHvaStart(0), IsSecArgPass(0), MemAlign(0), OrigAlign(0),
IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) {
static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big");
@@ -85,6 +86,9 @@ namespace ISD {
bool isInReg() const { return IsInReg; }
void setInReg() { IsInReg = 1; }
+ bool isArgInReg() const { return IsArgInReg; }
+ void setArgInReg() { IsArgInReg = 1; }
+
bool isSRet() const { return IsSRet; }
void setSRet() { IsSRet = 1; }
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index df0fced112f73..d2b2e502268fb 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -309,6 +309,7 @@ class LLVM_ABI TargetLoweringBase {
bool IsZExt : 1;
bool IsNoExt : 1;
bool IsInReg : 1;
+ bool IsArgInReg : 1;
bool IsSRet : 1;
bool IsNest : 1;
bool IsByVal : 1;
@@ -325,10 +326,11 @@ class LLVM_ABI TargetLoweringBase {
ArgListEntry(Value *Val, SDValue Node, Type *Ty)
: Val(Val), Node(Node), OrigTy(Ty), Ty(Ty), IsSExt(false),
- IsZExt(false), IsNoExt(false), IsInReg(false), IsSRet(false),
- IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
- IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
- IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {}
+ IsZExt(false), IsNoExt(false), IsInReg(false), IsArgInReg(false),
+ IsSRet(false), IsNest(false), IsByVal(false), IsByRef(false),
+ IsInAlloca(false), IsPreallocated(false), IsReturned(false),
+ IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false),
+ IsCFGuardTarget(false) {}
explicit ArgListEntry(Value *Val, SDValue Node = SDValue())
: ArgListEntry(Val, Node, Val->getType()) {}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 63c9d193421ea..32f1ad6763db4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -11323,6 +11323,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
// Set InReg Flag
Flags.setInReg();
}
+ if (Args[i].IsArgInReg)
+ Flags.setArgInReg();
if (Args[i].IsSRet)
Flags.setSRet();
if (Args[i].IsSwiftSelf)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index edba0a7169c0a..ad6b2cf421ba7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -134,6 +134,9 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = Call->getParamStackAlign(ArgIdx);
IndirectType = nullptr;
+ Value *Op = Call->getArgOperand(ArgIdx);
+ if (const Argument *Arg = dyn_cast<Argument>(Op))
+ IsArgInReg = Arg->hasInRegAttr();
assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 &&
"multiple ABI attributes?");
if (IsByVal) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 18f77ce2d76ec..db5364fbb4ad6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -4391,8 +4391,10 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
unsigned ArgIdx = 0;
for (auto [Reg, Val] : RegsToPass) {
- if (ArgIdx++ >= NumSpecialInputs &&
- (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
+ if (ArgIdx >= NumSpecialInputs &&
+ (IsChainCallConv || !Val->isDivergent() ||
+ Outs[ArgIdx - NumSpecialInputs].Flags.isArgInReg()) &&
+ TRI->isSGPRPhysReg(Reg)) {
// For chain calls, the inreg arguments are required to be
// uniform. Speculatively Insert a readfirstlane in case we cannot prove
// they are uniform.
@@ -4412,6 +4414,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue);
InGlue = Chain.getValue(1);
+ ArgIdx++;
}
// We don't usually want to end the call-sequence here because we would tidy
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll
deleted file mode 100644
index 34f4476f7fd6a..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -enable-var-scope %s
-
-; CHECK: illegal VGPR to SGPR copy
-
-declare hidden void @external_void_func_a15i32_inreg([15 x i32] inreg) #0
-declare hidden void @external_void_func_a16i32_inreg([16 x i32] inreg) #0
-declare hidden void @external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg, i32 inreg) #0
-
-define void @test_call_external_void_func_a15i32_inreg([15 x i32] inreg %arg0) #0 {
- call void @external_void_func_a15i32_inreg([15 x i32] inreg %arg0)
- ret void
-}
-
-define void @test_call_external_void_func_a16i32_inreg([16 x i32] inreg %arg0) #0 {
- call void @external_void_func_a16i32_inreg([16 x i32] inreg %arg0)
- ret void
-}
-
-define void @test_call_external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg %arg0, i32 inreg %arg1) #0 {
- call void @external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg %arg0, i32 inreg %arg1)
- ret void
-}
-
-attributes #0 = { nounwind }
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"amdhsa_code_object_version", i32 400}
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill.ll
new file mode 100644
index 0000000000000..fae5fa5ceeef7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope %s
+
+declare hidden void @external_void_func_a15i32_inreg([15 x i32] inreg) #0
+declare hidden void @external_void_func_a16i32_inreg([16 x i32] inreg) #0
+declare hidden void @external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg, i32 inreg) #0
+
+define void @test_call_external_void_func_a15i32_inreg([15 x i32] inreg %arg0) #0 {
+; CHECK-LABEL: test_call_external_void_func_a15i32_inreg:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s40, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[42:43], -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[42:43]
+; CHECK-NEXT: v_writelane_b32 v40, s40, 2
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: s_mov_b32 s40, s26
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: v_readfirstlane_b32 s26, v0
+; CHECK-NEXT: s_getpc_b64 s[42:43]
+; CHECK-NEXT: s_add_u32 s42, s42, external_void_func_a15i32_inreg at rel32@lo+4
+; CHECK-NEXT: s_addc_u32 s43, s43, external_void_func_a15i32_inreg at rel32@hi+12
+; CHECK-NEXT: s_mov_b32 s3, s19
+; CHECK-NEXT: s_mov_b32 s2, s18
+; CHECK-NEXT: s_mov_b32 s1, s17
+; CHECK-NEXT: s_mov_b32 s0, s16
+; CHECK-NEXT: s_mov_b32 s16, s20
+; CHECK-NEXT: s_mov_b32 s17, s21
+; CHECK-NEXT: s_mov_b32 s18, s22
+; CHECK-NEXT: s_mov_b32 s19, s23
+; CHECK-NEXT: s_mov_b32 s20, s24
+; CHECK-NEXT: s_mov_b32 s21, s25
+; CHECK-NEXT: s_mov_b32 s22, s40
+; CHECK-NEXT: s_mov_b32 s23, s27
+; CHECK-NEXT: s_mov_b32 s24, s28
+; CHECK-NEXT: s_mov_b32 s25, s29
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[42:43]
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v40, 2
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ call void @external_void_func_a15i32_inreg([15 x i32] inreg %arg0)
+ ret void
+}
+
+define void @test_call_external_void_func_a16i32_inreg([16 x i32] inreg %arg0) #0 {
+; CHECK-LABEL: test_call_external_void_func_a16i32_inreg:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s40, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[42:43], -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[42:43]
+; CHECK-NEXT: v_writelane_b32 v40, s40, 2
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: s_mov_b32 s41, s27
+; CHECK-NEXT: s_mov_b32 s40, s26
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: v_readfirstlane_b32 s26, v0
+; CHECK-NEXT: v_readfirstlane_b32 s27, v1
+; CHECK-NEXT: s_getpc_b64 s[42:43]
+; CHECK-NEXT: s_add_u32 s42, s42, external_void_func_a16i32_inreg at rel32@lo+4
+; CHECK-NEXT: s_addc_u32 s43, s43, external_void_func_a16i32_inreg at rel32@hi+12
+; CHECK-NEXT: s_mov_b32 s3, s19
+; CHECK-NEXT: s_mov_b32 s2, s18
+; CHECK-NEXT: s_mov_b32 s1, s17
+; CHECK-NEXT: s_mov_b32 s0, s16
+; CHECK-NEXT: s_mov_b32 s16, s20
+; CHECK-NEXT: s_mov_b32 s17, s21
+; CHECK-NEXT: s_mov_b32 s18, s22
+; CHECK-NEXT: s_mov_b32 s19, s23
+; CHECK-NEXT: s_mov_b32 s20, s24
+; CHECK-NEXT: s_mov_b32 s21, s25
+; CHECK-NEXT: s_mov_b32 s22, s40
+; CHECK-NEXT: s_mov_b32 s23, s41
+; CHECK-NEXT: s_mov_b32 s24, s28
+; CHECK-NEXT: s_mov_b32 s25, s29
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[42:43]
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v40, 2
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ call void @external_void_func_a16i32_inreg([16 x i32] inreg %arg0)
+ ret void
+}
+
+define void @test_call_external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg %arg0, i32 inreg %arg1) #0 {
+; CHECK-LABEL: test_call_external_void_func_a15i32_inreg_i32_inreg:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s40, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[42:43], -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[42:43]
+; CHECK-NEXT: v_writelane_b32 v40, s40, 2
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: s_mov_b32 s41, s27
+; CHECK-NEXT: s_mov_b32 s40, s26
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: v_readfirstlane_b32 s26, v0
+; CHECK-NEXT: v_readfirstlane_b32 s27, v1
+; CHECK-NEXT: s_getpc_b64 s[42:43]
+; CHECK-NEXT: s_add_u32 s42, s42, external_void_func_a15i32_inreg_i32_inreg at rel32@lo+4
+; CHECK-NEXT: s_addc_u32 s43, s43, external_void_func_a15i32_inreg_i32_inreg at rel32@hi+12
+; CHECK-NEXT: s_mov_b32 s3, s19
+; CHECK-NEXT: s_mov_b32 s2, s18
+; CHECK-NEXT: s_mov_b32 s1, s17
+; CHECK-NEXT: s_mov_b32 s0, s16
+; CHECK-NEXT: s_mov_b32 s16, s20
+; CHECK-NEXT: s_mov_b32 s17, s21
+; CHECK-NEXT: s_mov_b32 s18, s22
+; CHECK-NEXT: s_mov_b32 s19, s23
+; CHECK-NEXT: s_mov_b32 s20, s24
+; CHECK-NEXT: s_mov_b32 s21, s25
+; CHECK-NEXT: s_mov_b32 s22, s40
+; CHECK-NEXT: s_mov_b32 s23, s41
+; CHECK-NEXT: s_mov_b32 s24, s28
+; CHECK-NEXT: s_mov_b32 s25, s29
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[42:43]
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v40, 2
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ call void @external_void_func_a15i32_inreg_i32_inreg([15 x i32] inreg %arg0, i32 inreg %arg1)
+ ret void
+}
+
+attributes #0 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 400}
``````````
</details>
https://github.com/llvm/llvm-project/pull/178198
More information about the llvm-commits
mailing list