[llvm-branch-commits] [llvm] [AMDGPU] Fix async operations in GlobalISel on gfx12-plus (PR #190776)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Apr 7 04:22:57 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Sameer Sahasrabuddhe (ssahasra)
<details>
<summary>Changes</summary>
For GFX1250 async LDS intrinsics, map the LDS pointer operand to VGPR instead of SGPR. These instructions use $vdst/$vdata (VGPROp_32) for the LDS address, unlike the pre-GFX12 variants which use M0 (SGPR).
Assisted-By: Claude Opus 4.6
---
Patch is 57.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/190776.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp (+12-3)
- (modified) llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll (+365-174)
- (modified) llvm/test/CodeGen/AMDGPU/asyncmark-max-pregfx12.ll (+6-3)
- (modified) llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll (+423-207)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index f14cc8e0446dc..75c4ff2ef28df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -5556,7 +5556,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
- OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ // LDS address goes into $vdst (VGPR).
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
unsigned M0Bank =
getRegBankID(MI.getOperand(5).getReg(), MRI, AMDGPU::SGPRRegBankID);
OpdsMapping[5] = AMDGPU::getValueMapping(M0Bank, 32);
@@ -5569,10 +5570,18 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case Intrinsic::amdgcn_global_load_async_to_lds_b8:
case Intrinsic::amdgcn_global_load_async_to_lds_b32:
case Intrinsic::amdgcn_global_load_async_to_lds_b64:
- case Intrinsic::amdgcn_global_load_async_to_lds_b128:
+ case Intrinsic::amdgcn_global_load_async_to_lds_b128: {
+ OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ // LDS address goes into $vdst/$vdata (VGPR).
+ OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+ break;
+ }
case Intrinsic::amdgcn_load_to_lds:
- case Intrinsic::amdgcn_global_load_lds: {
+ case Intrinsic::amdgcn_load_async_to_lds:
+ case Intrinsic::amdgcn_global_load_lds:
+ case Intrinsic::amdgcn_global_load_async_lds: {
OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+ // LDS address goes into M0 (SGPR).
OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index cfb296fb2d529..4429468a3e0aa 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -1,48 +1,90 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GFX1250
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GISEL
; Test async mark/wait with global_load_lds and global loads
; This version uses wave barriers to enforce program order so that unrelated vmem
; instructions do not get reordered before reaching this point.
define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
-; GFX1250-LABEL: interleaved_with_wave_barrier:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
-; GFX1250-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
-; GFX1250-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
-; GFX1250-NEXT: global_load_b32 v10, v[8:9], off offset:44
-; GFX1250-NEXT: global_load_b32 v11, v[0:1], off offset:4
-; GFX1250-NEXT: ; wave barrier
-; GFX1250-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
-; GFX1250-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
-; GFX1250-NEXT: ; wave barrier
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: global_load_b32 v0, v[0:1], off offset:8
-; GFX1250-NEXT: ; wave barrier
-; GFX1250-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
-; GFX1250-NEXT: ; wave barrier
-; GFX1250-NEXT: global_load_b32 v1, v[8:9], off offset:48
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(1)
-; GFX1250-NEXT: s_wait_asynccnt 0x1
-; GFX1250-NEXT: ds_load_b32 v3, v2 offset:84
-; GFX1250-NEXT: ; wait_asyncmark(0)
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: ds_load_b32 v2, v2 offset:88
-; GFX1250-NEXT: s_wait_loadcnt 0x2
-; GFX1250-NEXT: v_add_nc_u32_e32 v4, v11, v10
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x101
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add3_u32 v0, v4, v3, v0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_add3_u32 v0, v0, v1, v2
-; GFX1250-NEXT: global_store_b32 v[6:7], v0, off
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; SDAG-LABEL: interleaved_with_wave_barrier:
+; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
+; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
+; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
+; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
+; SDAG-NEXT: global_load_b32 v10, v[8:9], off offset:44
+; SDAG-NEXT: global_load_b32 v11, v[0:1], off offset:4
+; SDAG-NEXT: ; wave barrier
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
+; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
+; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
+; SDAG-NEXT: ; wave barrier
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; SDAG-NEXT: ; wave barrier
+; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
+; SDAG-NEXT: ; wave barrier
+; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_wait_asynccnt 0x1
+; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
+; SDAG-NEXT: s_wait_loadcnt 0x2
+; SDAG-NEXT: v_add_nc_u32_e32 v4, v11, v10
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_add3_u32 v0, v4, v3, v0
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_add3_u32 v0, v0, v1, v2
+; SDAG-NEXT: global_store_b32 v[6:7], v0, off
+; SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GISEL-LABEL: interleaved_with_wave_barrier:
+; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
+; GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x54, v0
+; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
+; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GISEL-NEXT: global_load_b32 v10, v[8:9], off offset:44
+; GISEL-NEXT: global_load_b32 v11, v[0:1], off offset:4
+; GISEL-NEXT: ; wave barrier
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
+; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
+; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
+; GISEL-NEXT: ; wave barrier
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; GISEL-NEXT: ; wave barrier
+; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
+; GISEL-NEXT: ; wave barrier
+; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_wait_asynccnt 0x1
+; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
+; GISEL-NEXT: s_wait_loadcnt 0x2
+; GISEL-NEXT: v_add_nc_u32_e32 v6, v11, v10
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GISEL-NEXT: v_add3_u32 v0, v6, v3, v0
+; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT: v_add3_u32 v0, v0, v1, v2
+; GISEL-NEXT: global_store_b32 v[4:5], v0, off
+; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
; First batch: global load, global load, async global-to-LDS
%bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11
@@ -92,58 +134,116 @@ entry:
; those outstanding operations.
define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
-; GFX1250-LABEL: test_pipelined_loop:
-; GFX1250: ; %bb.0: ; %prolog
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: s_add_co_i32 s6, s2, 4
-; GFX1250-NEXT: s_mov_b32 s7, s2
-; GFX1250-NEXT: v_mov_b32_e32 v2, s6
-; GFX1250-NEXT: s_mov_b32 s6, 2
-; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v1, 4
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: .LBB1_1: ; %loop_body
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_add_co_i32 s8, s7, 8
-; GFX1250-NEXT: s_add_co_i32 s6, s6, 1
-; GFX1250-NEXT: v_mov_b32_e32 v2, s8
-; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, s7
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(2)
-; GFX1250-NEXT: s_wait_asynccnt 0x2
-; GFX1250-NEXT: s_add_co_i32 s7, s7, 4
-; GFX1250-NEXT: s_cmp_lt_i32 s6, s3
-; GFX1250-NEXT: ds_load_b32 v2, v2
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1250-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1250-NEXT: ; %bb.2: ; %epilog
-; GFX1250-NEXT: s_lshl2_add_u32 s0, s3, s2
-; GFX1250-NEXT: ; wait_asyncmark(1)
-; GFX1250-NEXT: s_wait_asynccnt 0x1
-; GFX1250-NEXT: s_add_co_i32 s0, s0, -8
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
-; GFX1250-NEXT: ds_load_b32 v0, v0
-; GFX1250-NEXT: ; wait_asyncmark(0)
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b32 v2, v0, s[0:1]
-; GFX1250-NEXT: s_endpgm
+; SDAG-LABEL: test_pipelined_loop:
+; SDAG: ; %bb.0: ; %prolog
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-NEXT: s_clause 0x1
+; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; SDAG-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-NEXT: s_add_co_i32 s6, s2, 4
+; SDAG-NEXT: s_mov_b32 s7, s2
+; SDAG-NEXT: v_mov_b32_e32 v2, s6
+; SDAG-NEXT: s_mov_b32 s6, 2
+; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
+; SDAG-NEXT: v_mov_b32_e32 v1, 4
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1] offset:4 nv
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: .LBB1_1: ; %loop_body
+; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: s_add_co_i32 s8, s7, 8
+; SDAG-NEXT: s_add_co_i32 s6, s6, 1
+; SDAG-NEXT: v_mov_b32_e32 v2, s8
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
+; SDAG-NEXT: v_mov_b32_e32 v2, s7
+; SDAG-NEXT: ; asyncmark
+; SDAG-NEXT: ; wait_asyncmark(2)
+; SDAG-NEXT: s_wait_asynccnt 0x2
+; SDAG-NEXT: s_add_co_i32 s7, s7, 4
+; SDAG-NEXT: s_cmp_lt_i32 s6, s3
+; SDAG-NEXT: ds_load_b32 v2, v2
+; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
+; SDAG-NEXT: ; %bb.2: ; %epilog
+; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
+; SDAG-NEXT: ; wait_asyncmark(1)
+; SDAG-NEXT: s_wait_asynccnt 0x1
+; SDAG-NEXT: s_add_co_i32 s0, s0, -8
+; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
+; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
+; SDAG-NEXT: ds_load_b32 v0, v0
+; SDAG-NEXT: ; wait_asyncmark(0)
+; SDAG-NEXT: s_wait_dscnt 0x0
+; SDAG-NEXT: s_wait_asynccnt 0x0
+; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
+; SDAG-NEXT: s_wait_kmcnt 0x0
+; SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
+; SDAG-NEXT: s_endpgm
+;
+; GISEL-LABEL: test_pipelined_loop:
+; GISEL: ; %bb.0: ; %prolog
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-NEXT: s_clause 0x1
+; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GISEL-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
+; GISEL-NEXT: s_mov_b32 s7, 2
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, s7
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v0, s2
+; GISEL-NEXT: s_add_co_u32 s6, s2, 4
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4 nv
+; GISEL-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, 4
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: s_mov_b32 s6, 0
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:4 nv
+; GISEL-NEXT: s_add_co_u32 s0, s0, 8
+; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GISEL-NEXT: .LBB1_1: ; %loop_body
+; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: v_dual_add_nc_u32 v5, 8, v3 :: v_dual_add_nc_u32 v4, 1, v4
+; GISEL-NEXT: global_load_async_to_lds_b32 v5, v[0:1], off offset:4 nv
+; GISEL-NEXT: ; asyncmark
+; GISEL-NEXT: ; wait_asyncmark(2)
+; GISEL-NEXT: s_wait_asynccnt 0x2
+; GISEL-NEXT: ds_load_b32 v5, v3
+; GISEL-NEXT: v_add_co_u32 v0, s0, v0, 4
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s0
+; GISEL-NEXT: v_add_nc_u32_e32 v3, 4, v3
+; GISEL-NEXT: v_cmp_gt_i32_e32 vcc_lo, s3, v4
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GISEL-NEXT: s_cbranch_vccnz .LBB1_1
+; GISEL-NEXT: ; %bb.2: ; %epilog
+; GISEL-NEXT: s_lshl_b32 s0, s3, 2
+; GISEL-NEXT: ; wait_asyncmark(1)
+; GISEL-NEXT: s_wait_asynccnt 0x1
+; GISEL-NEXT: s_add_co_u32 s0, s2, s0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: s_add_co_u32 s0, s0, -8
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
+; GISEL-NEXT: ds_load_b32 v0, v0
+; GISEL-NEXT: ; wait_asyncmark(0)
+; GISEL-NEXT: s_wait_dscnt 0x0
+; GISEL-NEXT: s_wait_asynccnt 0x0
+; GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0
+; GISEL-NEXT: s_wait_kmcnt 0x0
+; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-NEXT: s_endpgm
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
@@ -201,90 +301,181 @@ epilog:
; Software pipelined loop with async global-to-LDS and global loads
define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
-; GFX1250-LABEL: test_pipelined_loop_with_global:
-; GFX1250: ; %bb.0: ; %prolog
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_b32 s6, s[8:9], 0x0
-; GFX1250-NEXT: s_load_b32 s7, s[0:1], 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s10
-; GFX1250-NEXT: s_add_co_i32 s11, s10, 4
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11
-; GFX1250-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
-; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
-; GFX1250-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
-; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7
-; GFX1250-NEXT: s_mov_b64 s[6:7], s[2:3]
-; GFX1250-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
-; GFX1250-NEXT: s_mov_b32 s8, 2
-; GFX1250-NEXT: s_mov_b32 s9, s10
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: .LBB2_1: ; %loop_body
-; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_add_co_i32 s12, s9, 8
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
-; GFX1250-NEXT: v_mov_b32_e32 v8, v3
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_b32 v3, v0, s[4:5]
-; GFX1250-NEXT: global_load_b32 v4, v0, s[0:1]
-; GFX1250-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
-; GFX1250-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v9, s9
-; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: ; wait_asyncmark(2)
-; GFX1250-NEXT: s_wait_asynccnt 0x2
-; GFX1250-NEXT: s_wait_asynccnt 0x2
-; GFX1250-NEXT: s_add_co_i32 s8, s8, 1
-; GFX1250-NEXT: s_add_co_i32 s9, s9, 4
-; GFX1250-NEXT: ds_load_b32 v9, v9
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
-; GFX1250-NEXT: s_cmp_lt_i32 s8, s11
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
-; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[4:5], 4
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v9, v10, v9
-; GFX1250-NEXT: global_store_b32 v0, v9, s[6:7]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4
-; GFX1250-NEXT: s_cbranch_scc1 .LBB2_1
-; GFX1250-NEXT: ; %bb.2: ; %epilog
-; GFX1250-NEXT: s_add_co_i32 s0, s11, -2
-; GFX1250-NEXT: ; wait_asyncmark(1)
-; GFX1250-NEXT: s_wait_asynccnt 0x1
-; GFX1250-NEXT: s_lshl2_add_u32 s1, s0, s10
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1
-; GFX1250-NEXT: ds_load_b32 v1, v0
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v2, v1
-; GFX1250-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
-; GFX1250-NEXT: ; wait_asyncmark(0)
-; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: ds_load_b32 v0, v0 offset:4
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v4
-; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u32_e32 v0,...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/190776
More information about the llvm-branch-commits
mailing list