[llvm] [AMDGPU] Do not scale private alloca size when using flat-scratch (PR #201142)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 2 08:06:50 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Juan Manuel Martinez Caamaño (jmmartinez)
<details>
<summary>Changes</summary>
When using flat-scratch, the `scratch_load/scratch_store` instructions scale the stack offset by the wavefront size on their own.
Scaling the alloca-size by the wave-front size lead to accesses outside of the private-memory limit.
---
Let's consider the following LLVM-IR which allocates 2 private arrays of 4xi32 each:
```llvm
define amdgpu_kernel void @<!-- -->kernel() {
entry:
br label %body
body:
%A = alloca [4 x i32], addrspace(5)
%a.0 = getelementptr [4 x i32], ptr addrspace(5) %A, i32 0, i32 0
%a.1 = getelementptr [4 x i32], ptr addrspace(5) %A, i32 0, i32 1
store volatile i32 3, ptr addrspace(5) %a.0
store volatile i32 5, ptr addrspace(5) %a.1
%B = alloca [4 x i32], addrspace(5)
%b.0 = getelementptr [4 x i32], ptr addrspace(5) %B, i32 0, i32 0
%b.1 = getelementptr [4 x i32], ptr addrspace(5) %B, i32 0, i32 1
store volatile i32 7, ptr addrspace(5) %b.0
store volatile i32 9, ptr addrspace(5) %b.1
ret void
}
```
We can compile it with these two configs:
* Without flat-scratch: `clang -O0 -x ir -target amdgcn-amd-amdhsa -mcpu=gfx90a -shared -o kernel-noflat.hsaco -Xclang -target-feature -Xclang -enable-flat-scratch`
* With flat-scratch: `clang -O0 -x ir -target amdgcn-amd-amdhsa -mcpu=gfx90a -shared -o kernel-flat.hsaco -Xclang -target-feature -Xclang +enable-flat-scratch`
Then, we can use a harness to run it:
```cpp
#include <hip/hip_runtime.h>
#include <cstdio>
#include <cstdlib>
int main(int argc, char** argv) {
const char* hsaco = (argc > 1) ? argv[1] : "build/a.hsaco";
hipModule_t module{};
hipFunction_t func{};
hipError_t err = hipModuleLoad(&module, hsaco);
if (err != hipSuccess) {
std::fprintf(stderr, "hipModuleLoad(%s): %s\n", hsaco, hipGetErrorString(err));
return 1;
}
err = hipModuleGetFunction(&func, module, "kernel");
if (err != hipSuccess) {
std::fprintf(stderr, "hipModuleGetFunction(kernel): %s\n", hipGetErrorString(err));
return 1;
}
err = hipModuleLaunchKernel(func, 1, 1, 1, 64, 1, 1, 0, nullptr, nullptr, nullptr);
if (err != hipSuccess) {
std::fprintf(stderr, "hipModuleLaunchKernel: %s\n", hipGetErrorString(err));
return 1;
}
err = hipDeviceSynchronize();
if (err != hipSuccess) {
std::fprintf(stderr, "hipDeviceSynchronize: %s\n", hipGetErrorString(err));
return 1;
}
std::printf("kernel launched OK\n");
return 0;
}
```
Inside `rocgdb`, I set a breakpoint in `kernel` and run, then step until the `s_endpgm`. I then dump the contents of the private memory for the thread-0 using `x/32x private_lane#<!-- -->0`.
This shows for noflat:
```
private_lane#<!-- -->0x0: 0x00000000 0x00000000 0x00000000 0x00000000
private_lane#<!-- -->0x10: 0x00000003 0x00000005 0x00000000 0x00000000
private_lane#<!-- -->0x20: 0x00000007 0x00000009 0x00000000 0x00000000
...
private_lane#<!-- -->0x70: 0x00000000 0x00000000 0x00000000 0x00000000
```
and for flat:
```
private_lane#<!-- -->0x0: 0x00000000 0x00000000 0x00000000 0x00000000
private_lane#<!-- -->0x10: 0x00000003 0x00000005 0x00000000 0x00000000
private_lane#<!-- -->0x20: 0x00000000 0x00000000 0x00000000 0x00000000
...
private_lane#<!-- -->0x70: 0x00000000 0x00000000 0x00000000 0x00000000
```
The instructions that store these values are in fact trying to access the offset `0x10+sizeof(i32)*4*wave_size == 0x410` which is already outside of the private memory limit. The correct offset to pass to `scratch_store` is `0x10+sizeof(i32)*4 == 0x20`.
After this patch the observed behavior with flat-scratch enabled/disabled is the same.
---
Patch is 61.30 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/201142.diff
7 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp (+16-4)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+22-9)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll (+5-16)
- (modified) llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll (+24-32)
- (modified) llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll (+99-134)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll (+4-8)
- (modified) llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll (+8-8)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index a24df782cf28a..540447890dfd2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1174,6 +1174,11 @@ bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
Register AllocSize = MI.getOperand(1).getReg();
Align Alignment = assumeAligned(MI.getOperand(2).getImm());
+ // When using flat-scratch, the stack offset is already scaled by the
+ // wave-size by the hardware instructions.
+ const bool HasFlatScratch = ST.hasFlatScratchEnabled();
+ const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
+
const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI);
if (SizeBank != &AMDGPU::SGPRRegBank) {
@@ -1191,16 +1196,23 @@ bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
Register SPReg = Info->getStackPtrOffsetReg();
ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
- auto WaveSize = B.buildConstant(LLT::scalar(32), ST.getWavefrontSizeLog2());
- auto ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize);
+ Register ScaledSize = AllocSize;
+ if (!HasFlatScratch) {
+ auto WaveSize = B.buildConstant(LLT::scalar(32), WavefrontSizeLog2);
+ ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize).getReg(0);
+ }
auto OldSP = B.buildCopy(PtrTy, SPReg);
if (Alignment > TFI.getStackAlign()) {
- auto StackAlignMask = (Alignment.value() << ST.getWavefrontSizeLog2()) - 1;
+ const uint64_t ScaledAlignment =
+ Alignment.value() << (HasFlatScratch ? 0u : WavefrontSizeLog2);
+ auto StackAlignMask = ScaledAlignment - 1;
auto Tmp1 = B.buildPtrAdd(PtrTy, OldSP,
B.buildConstant(LLT::scalar(32), StackAlignMask));
B.buildMaskLowPtrBits(Dst, Tmp1,
- Log2(Alignment) + ST.getWavefrontSizeLog2());
+ (HasFlatScratch
+ ? Log2(Alignment)
+ : Log2(Alignment) + WavefrontSizeLog2));
} else {
B.buildCopy(Dst, OldSP);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 86f2479490c29..4603e6cac6bde 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -4680,10 +4680,15 @@ SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
"Stack grows upwards for AMDGPU");
Chain = BaseAddr.getValue(1);
+ // When using flat-scratch, the stack offset is already is scaled by the
+ // wave-size by the hardware instructions.
+ const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
+ const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
+
Align StackAlign = TFL->getStackAlign();
if (Alignment > StackAlign) {
uint64_t ScaledAlignment = Alignment.value()
- << Subtarget->getWavefrontSizeLog2();
+ << (HasFlatScratch ? 0 : WavefrontSizeLog2);
uint64_t StackAlignMask = ScaledAlignment - 1;
SDValue TmpAddr = DAG.getNode(ISD::ADD, dl, VT, BaseAddr,
DAG.getConstant(StackAlignMask, dl, VT));
@@ -4694,21 +4699,29 @@ SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
SDValue NewSP;
if (isa<ConstantSDNode>(Size)) {
- // For constant sized alloca, scale alloca size by wave-size
- SDValue ScaledSize = DAG.getNode(
- ISD::SHL, dl, VT, Size,
- DAG.getConstant(Subtarget->getWavefrontSizeLog2(), dl, MVT::i32));
+ // Increase the stack pointer by the size of the alloca.
+ // If not using flat-scratch, we have to scale the size by the wave-size.
+ SDValue ScaledSize = Size;
+ if (!HasFlatScratch) {
+ ScaledSize =
+ DAG.getNode(ISD::SHL, dl, VT, Size,
+ DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
+ }
NewSP = DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value
} else {
// For dynamic sized alloca, perform wave-wide reduction to get max of
- // alloca size(divergent) and then scale it by wave-size
+ // alloca size(divergent), and then scale it (when not using flat-scratch)
+ // by wave-size.
SDValue WaveReduction =
DAG.getTargetConstant(Intrinsic::amdgcn_wave_reduce_umax, dl, MVT::i32);
Size = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32, WaveReduction,
Size, DAG.getTargetConstant(0, dl, MVT::i32));
- SDValue ScaledSize = DAG.getNode(
- ISD::SHL, dl, VT, Size,
- DAG.getConstant(Subtarget->getWavefrontSizeLog2(), dl, MVT::i32));
+ SDValue ScaledSize = Size;
+ if (!HasFlatScratch) {
+ ScaledSize =
+ DAG.getNode(ISD::SHL, dl, VT, Size,
+ DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
+ }
NewSP =
DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value in vgpr.
SDValue ReadFirstLaneID =
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
index 2e8f8f67bc5ea..aaff0fc4848ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
@@ -54,8 +54,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align4(i32 %n) #0 {
; GFX11-NEXT: s_lshl2_add_u32 s1, s1, 15
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s1, s1, -16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s0, s1
; GFX11-NEXT: s_endpgm
%alloca = alloca i32, i32 %n, align 4, addrspace(5)
@@ -134,8 +132,6 @@ define void @func_dynamic_stackalloc_sgpr_align4() #0 {
; GFX11-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s0, s0, -16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s2, s0
; GFX11-NEXT: s_mov_b32 s32, s33
; GFX11-NEXT: s_mov_b32 s33, s3
@@ -195,8 +191,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align16(i32 %n) #0 {
; GFX11-NEXT: s_lshl2_add_u32 s1, s1, 15
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s1, s1, -16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s0, s1
; GFX11-NEXT: s_endpgm
%alloca = alloca i32, i32 %n, align 16, addrspace(5)
@@ -275,8 +269,6 @@ define void @func_dynamic_stackalloc_sgpr_align16() #0 {
; GFX11-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s0, s0, -16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s2, s0
; GFX11-NEXT: s_mov_b32 s32, s33
; GFX11-NEXT: s_mov_b32 s33, s3
@@ -331,16 +323,14 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) #0 {
; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-NEXT: s_mov_b32 s32, 32
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_add_u32 s1, s32, 0x3ff
+; GFX11-NEXT: s_add_u32 s1, s32, 31
; GFX11-NEXT: s_mov_b32 s33, 0
-; GFX11-NEXT: s_and_b32 s1, s1, 0xfffffc00
+; GFX11-NEXT: s_and_not1_b32 s1, s1, 31
; GFX11-NEXT: scratch_store_b32 off, v0, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s0, s0, -16
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s1, s0
; GFX11-NEXT: s_endpgm
%alloca = alloca i32, i32 %n, align 32, addrspace(5)
@@ -427,15 +417,14 @@ define void @func_dynamic_stackalloc_sgpr_align32(ptr addrspace(1) %out) #0 {
; GFX11-NEXT: s_mov_b32 s33, s2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX11-NEXT: s_add_u32 s1, s32, 0x3ff
+; GFX11-NEXT: s_add_u32 s1, s32, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, 0xfffffc00
+; GFX11-NEXT: s_and_not1_b32 s1, s1, 31
; GFX11-NEXT: scratch_store_b32 off, v0, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-NEXT: s_and_b32 s0, s0, -16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s0, s0, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_u32 s32, s1, s0
; GFX11-NEXT: s_mov_b32 s32, s34
; GFX11-NEXT: s_mov_b32 s34, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
index 0835837a98f98..851bd6d2f75b4 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
@@ -22,7 +22,7 @@ define amdgpu_cs_chain void @test_alloca() #0 {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s0, s32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s32, s0, 0x200
+; GFX12-NEXT: s_add_co_i32 s32, s0, 16
; GFX12-NEXT: scratch_store_b32 off, v0, s0
; GFX12-NEXT: s_endpgm
;
@@ -33,7 +33,7 @@ define amdgpu_cs_chain void @test_alloca() #0 {
; GFX942-NEXT: s_add_i32 s32, s32, 16
; GFX942-NEXT: s_mov_b32 s0, s32
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_add_i32 s32, s0, 0x400
+; GFX942-NEXT: s_add_i32 s32, s0, 16
; GFX942-NEXT: scratch_store_dword off, v0, s0
; GFX942-NEXT: s_endpgm
.entry:
@@ -53,15 +53,13 @@ define amdgpu_cs_chain void @test_alloca_var_uniform(i32 inreg %count) #0 {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: s_mov_b32 s33, s32
; GFX12-NEXT: s_add_co_i32 s32, s32, 16
+; GFX12-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s0, s0, -16
; GFX12-NEXT: s_mov_b32 s1, s32
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s0, s0, 5
+; GFX12-NEXT: s_and_b32 s0, s0, -16
; GFX12-NEXT: scratch_store_b32 off, v0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s32, s1, s0
@@ -70,11 +68,10 @@ define amdgpu_cs_chain void @test_alloca_var_uniform(i32 inreg %count) #0 {
; GFX942-LABEL: test_alloca_var_uniform:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX942-NEXT: s_mov_b32 s33, s32
; GFX942-NEXT: s_add_i32 s32, s32, 16
+; GFX942-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX942-NEXT: s_and_b32 s0, s0, -16
-; GFX942-NEXT: s_lshl_b32 s0, s0, 6
; GFX942-NEXT: s_mov_b32 s1, s32
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_add_i32 s32, s1, s0
@@ -116,7 +113,7 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) #0 {
; GFX12-NEXT: s_mov_b32 s0, s32
; GFX12-NEXT: v_mov_b32_e32 v3, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshl_add_u32 v2, s1, 5, s0
+; GFX12-NEXT: v_add_nc_u32_e64 v2, s0, s1
; GFX12-NEXT: scratch_store_b32 off, v3, s0
; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_endpgm
@@ -146,8 +143,8 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) #0 {
; GFX942-NEXT: v_readlane_b32 s2, v0, 63
; GFX942-NEXT: s_mov_b64 exec, s[0:1]
; GFX942-NEXT: s_mov_b32 s0, s32
-; GFX942-NEXT: v_mov_b32_e32 v1, s0
-; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: v_add_u32_e32 v1, s0, v1
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_readfirstlane_b32 s32, v1
; GFX942-NEXT: v_mov_b32_e32 v1, 0
@@ -178,7 +175,7 @@ define amdgpu_cs_chain void @test_alloca_and_call() #0 {
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX12-NEXT: s_mov_b32 s2, s32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s32, s2, 0x200
+; GFX12-NEXT: s_add_co_i32 s32, s2, 16
; GFX12-NEXT: scratch_store_b32 off, v0, s2
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -196,7 +193,7 @@ define amdgpu_cs_chain void @test_alloca_and_call() #0 {
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX942-NEXT: s_mov_b32 s2, s32
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_add_i32 s32, s2, 0x400
+; GFX942-NEXT: s_add_i32 s32, s2, 16
; GFX942-NEXT: scratch_store_dword off, v0, s2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
@@ -227,16 +224,15 @@ define amdgpu_cs_chain void @test_alloca_and_call_var_uniform(i32 inreg %count)
; GFX12-NEXT: s_add_co_u32 s2, s2, foo at gotpcrel32@lo+12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s3, s3, foo at gotpcrel32@hi+24
-; GFX12-NEXT: s_lshl2_add_u32 s0, s0, 15
-; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_and_b32 s0, s0, -16
+; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
+; GFX12-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX12-NEXT: s_mov_b32 s1, s32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s0, s0, 5
+; GFX12-NEXT: s_and_b32 s0, s0, -16
+; GFX12-NEXT: scratch_store_b32 off, v0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s32, s1, s0
-; GFX12-NEXT: scratch_store_b32 off, v0, s1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_swappc_b64 s[30:31], s[2:3]
@@ -246,10 +242,9 @@ define amdgpu_cs_chain void @test_alloca_and_call_var_uniform(i32 inreg %count)
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_lshl2_add_u32 s0, s0, 15
-; GFX942-NEXT: s_and_b32 s0, s0, -16
; GFX942-NEXT: s_mov_b32 s33, s32
; GFX942-NEXT: s_add_i32 s32, s32, 16
-; GFX942-NEXT: s_lshl_b32 s2, s0, 6
+; GFX942-NEXT: s_and_b32 s2, s0, -16
; GFX942-NEXT: s_getpc_b64 s[0:1]
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
@@ -308,7 +303,7 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) #0 {
; GFX12-NEXT: s_mov_b32 s2, s32
; GFX12-NEXT: v_mov_b32_e32 v3, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshl_add_u32 v2, s3, 5, s2
+; GFX12-NEXT: v_add_nc_u32_e64 v2, s2, s3
; GFX12-NEXT: scratch_store_b32 off, v3, s2
; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_wait_kmcnt 0x0
@@ -345,8 +340,8 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) #0 {
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX942-NEXT: s_mov_b32 s3, s32
-; GFX942-NEXT: v_mov_b32_e32 v1, s3
-; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: v_add_u32_e32 v1, s3, v1
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_readfirstlane_b32 s32, v1
; GFX942-NEXT: v_mov_b32_e32 v1, 0
@@ -378,7 +373,7 @@ define amdgpu_cs_chain void @test_call_and_alloca() #0 {
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
; GFX12-NEXT: s_mov_b32 s4, s32
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX12-NEXT: s_add_co_i32 s32, s4, 0x200
+; GFX12-NEXT: s_add_co_i32 s32, s4, 16
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_swappc_b64 s[30:31], s[0:1]
@@ -396,7 +391,7 @@ define amdgpu_cs_chain void @test_call_and_alloca() #0 {
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX942-NEXT: s_mov_b32 s4, s32
-; GFX942-NEXT: s_add_i32 s32, s4, 0x400
+; GFX942-NEXT: s_add_i32 s32, s4, 16
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v0, 0
@@ -433,8 +428,6 @@ define amdgpu_cs_chain void @test_call_and_alloca_var_uniform(i32 inreg %count)
; GFX12-NEXT: s_and_b32 s0, s0, -16
; GFX12-NEXT: s_mov_b32 s4, s32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s0, s0, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s32, s4, s0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -447,10 +440,9 @@ define amdgpu_cs_chain void @test_call_and_alloca_var_uniform(i32 inreg %count)
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_lshl2_add_u32 s0, s0, 15
-; GFX942-NEXT: s_and_b32 s0, s0, -16
; GFX942-NEXT: s_mov_b32 s33, s32
; GFX942-NEXT: s_add_i32 s32, s32, 16
-; GFX942-NEXT: s_lshl_b32 s2, s0, 6
+; GFX942-NEXT: s_and_b32 s2, s0, -16
; GFX942-NEXT: s_getpc_b64 s[0:1]
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
@@ -508,7 +500,7 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) #0 {
; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_mov_b32 s4, s32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshl_add_u32 v2, s3, 5, s4
+; GFX12-NEXT: v_add_nc_u32_e64 v2, s4, s3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_wait_kmcnt 0x0
@@ -547,8 +539,8 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) #0 {
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX942-NEXT: s_mov_b32 s4, s32
-; GFX942-NEXT: v_mov_b32_e32 v1, s4
-; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: v_add_u32_e32 v1, s4, v1
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_readfirstlane_b32 s32, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 927da53d1664c..560ac268a6b10 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -55,8 +55,6 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform(i32 %n) #0 {
; GFX11-SDAG-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: s_and_b32 s0, s0, -16
-; GFX11-SDAG-NEXT: s_lshl_b32 s0, s0, 5
-; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: s_add_i32 s32, s1, s0
; GFX11-SDAG-NEXT: s_endpgm
;
@@ -73,8 +71,6 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform(i32 %n) #0 {
; GFX11-GISEL-NEXT: s_lshl2_add_u32 s1, s1, 15
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-GISEL-NEXT: s_and_b32 s1, s1, -16
-; GFX11-GISEL-NEXT: s_lshl_b32 s1, s1, 5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-GISEL-NEXT: s_add_u32 s32, s0, s1
; GFX11-GISEL-NEXT: s_endpgm
%alloca = alloca i32, i32 %n, addrspace(5)
@@ -128,17 +124,15 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_uniform_over_aligned(i
; GFX11-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-SDAG-NEXT: s_movk_i32 s32, 0x80
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 10
-; GFX11-SDAG-NEXT: s_add_i32 s1, s32, 0xfff
+; GFX11-SDAG-NEXT: s_add_i32 s1, s32, 0x7f
; GFX11-SDAG-NEXT: s_mov_b32 s33, 0
-; GFX11-SDAG-NEXT: s_and_b32 s1, s1, 0x...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/201142
More information about the llvm-commits
mailing list