[llvm] [AMDGPU] Add PC sign extension to insertIndirectBranch (PR #109233)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 18 21:59:59 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Carl Ritson (perlfu)
<details>
<summary>Changes</summary>
Fix missing sign extension of PC required on some ASICs.
Although this is unlikely to create issues in practice.
---
Patch is 43.53 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/109233.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+21)
- (modified) llvm/test/CodeGen/AMDGPU/branch-relaxation.ll (+973)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 30aa36be99c95f..034c4c8f295fb0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2873,9 +2873,28 @@ void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
auto I = MBB.end();
+ // Note: as this is used after hazard recognizer we need to apply some hazard
+ // workarounds directly.
+ const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
+ ST.hasVALUReadSGPRHazard();
+ auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
+ if (FlushSGPRWrites)
+ BuildMI(MBB, I, DL, get(AMDGPU::S_WAITCNT_DEPCTR))
+ .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
+ };
+
// We need to compute the offset relative to the instruction immediately after
// s_getpc_b64. Insert pc arithmetic code before last terminator.
MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
+ ApplyHazardWorkarounds();
+ if (ST.hasGetPCZeroExtension()) {
+ // Fix up hardware that does not sign-extend the 48-bit PC value by
+ // inserting: s_sext_i32_i16 reghi, reghi
+ BuildMI(MBB, I, DL, get(AMDGPU::S_SEXT_I32_I16))
+ .addReg(PCReg, RegState::Define, AMDGPU::sub1)
+ .addReg(PCReg, 0, AMDGPU::sub1);
+ ApplyHazardWorkarounds();
+ }
auto &MCCtx = MF->getContext();
MCSymbol *PostGetPCLabel =
@@ -2890,10 +2909,12 @@ void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
.addReg(PCReg, RegState::Define, AMDGPU::sub0)
.addReg(PCReg, 0, AMDGPU::sub0)
.addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);
+ ApplyHazardWorkarounds();
BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
.addReg(PCReg, RegState::Define, AMDGPU::sub1)
.addReg(PCReg, 0, AMDGPU::sub1)
.addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);
+ ApplyHazardWorkarounds();
// Insert the indirect branch after the other terminator.
BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
index 635f3e4886b875..132d5664cfc1df 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-s-branch-bits=4 -simplifycfg-require-and-preserve-domtree=1 -amdgpu-long-branch-factor=0 < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -amdgpu-s-branch-bits=5 -simplifycfg-require-and-preserve-domtree=1 -amdgpu-long-branch-factor=0 < %s | FileCheck -enable-var-scope -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -amdgpu-s-branch-bits=5 -simplifycfg-require-and-preserve-domtree=1 -amdgpu-long-branch-factor=0 < %s | FileCheck -enable-var-scope -check-prefix=GFX12 %s
; FIXME: We should use llvm-mc for this, but we can't even parse our own output.
@@ -42,6 +44,73 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
+;
+; GFX11-LABEL: uniform_conditional_max_short_forward_branch:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB0_1
+; GFX11-NEXT: ; %bb.3: ; %bb
+; GFX11-NEXT: s_getpc_b64 s[4:5]
+; GFX11-NEXT: .Lpost_getpc0:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB0_2-.Lpost_getpc0)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB0_2-.Lpost_getpc0)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[4:5]
+; GFX11-NEXT: .LBB0_1: ; %bb2
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: s_sleep 0
+; GFX11-NEXT: .LBB0_2: ; %bb3
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_nop 0
+; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: uniform_conditional_max_short_forward_branch:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cmp_eq_u32 s0, 0
+; GFX12-NEXT: s_cbranch_scc0 .LBB0_1
+; GFX12-NEXT: ; %bb.3: ; %bb
+; GFX12-NEXT: s_getpc_b64 s[4:5]
+; GFX12-NEXT: .Lpost_getpc0:
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_sext_i32_i16 s5, s5
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB0_2-.Lpost_getpc0)&4294967295
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB0_2-.Lpost_getpc0)>>32
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_setpc_b64 s[4:5]
+; GFX12-NEXT: .LBB0_1: ; %bb2
+; GFX12-NEXT: ;;#ASMSTART
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: ;;#ASMEND
+; GFX12-NEXT: s_sleep 0
+; GFX12-NEXT: .LBB0_2: ; %bb3
+; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
bb:
%cmp = icmp eq i32 %cnd, 0
br i1 %cmp, label %bb3, label %bb2 ; +8 dword branch
@@ -89,6 +158,73 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
+;
+; GFX11-LABEL: uniform_conditional_min_long_forward_branch:
+; GFX11: ; %bb.0: ; %bb0
+; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB1_1
+; GFX11-NEXT: ; %bb.3: ; %bb0
+; GFX11-NEXT: s_getpc_b64 s[4:5]
+; GFX11-NEXT: .Lpost_getpc1:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB1_2-.Lpost_getpc1)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB1_2-.Lpost_getpc1)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[4:5]
+; GFX11-NEXT: .LBB1_1: ; %bb2
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: .LBB1_2: ; %bb3
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_nop 0
+; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: uniform_conditional_min_long_forward_branch:
+; GFX12: ; %bb.0: ; %bb0
+; GFX12-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cmp_eq_u32 s0, 0
+; GFX12-NEXT: s_cbranch_scc0 .LBB1_1
+; GFX12-NEXT: ; %bb.3: ; %bb0
+; GFX12-NEXT: s_getpc_b64 s[4:5]
+; GFX12-NEXT: .Lpost_getpc1:
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_sext_i32_i16 s5, s5
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB1_2-.Lpost_getpc1)&4294967295
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB1_2-.Lpost_getpc1)>>32
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_setpc_b64 s[4:5]
+; GFX12-NEXT: .LBB1_1: ; %bb2
+; GFX12-NEXT: ;;#ASMSTART
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: ;;#ASMEND
+; GFX12-NEXT: .LBB1_2: ; %bb3
+; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
bb0:
%cmp = icmp eq i32 %cnd, 0
br i1 %cmp, label %bb3, label %bb2 ; +9 dword branch
@@ -138,6 +274,80 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
+;
+; GFX11-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
+; GFX11: ; %bb.0: ; %bb0
+; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_cmp_eq_f32_e64 s[4:5], s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_and_b64 vcc, exec, s[4:5]
+; GFX11-NEXT: s_cbranch_vccz .LBB2_1
+; GFX11-NEXT: ; %bb.3: ; %bb0
+; GFX11-NEXT: s_getpc_b64 s[4:5]
+; GFX11-NEXT: .Lpost_getpc2:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB2_2-.Lpost_getpc2)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB2_2-.Lpost_getpc2)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[4:5]
+; GFX11-NEXT: .LBB2_1: ; %bb2
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: ; 32 bytes
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: .LBB2_2: ; %bb3
+; GFX11-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_nop 0
+; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
+; GFX12: ; %bb.0: ; %bb0
+; GFX12-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cmp_eq_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s1, -1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 vcc_lo, exec_lo, s1
+; GFX12-NEXT: s_cbranch_vccz .LBB2_1
+; GFX12-NEXT: ; %bb.3: ; %bb0
+; GFX12-NEXT: s_getpc_b64 s[4:5]
+; GFX12-NEXT: .Lpost_getpc2:
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_sext_i32_i16 s5, s5
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB2_2-.Lpost_getpc2)&4294967295
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB2_2-.Lpost_getpc2)>>32
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_setpc_b64 s[4:5]
+; GFX12-NEXT: .LBB2_1: ; %bb2
+; GFX12-NEXT: ;;#ASMSTART
+; GFX12-NEXT: ; 32 bytes
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: ;;#ASMEND
+; GFX12-NEXT: .LBB2_2: ; %bb3
+; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x24
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
bb0:
%cmp = fcmp oeq float %cnd, 0.0
br i1 %cmp, label %bb3, label %bb2 ; + 8 dword branch
@@ -193,6 +403,87 @@ define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
; GCN-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
+;
+; GFX11-LABEL: min_long_forward_vbranch:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v2, v0, s[0:1] glc dlc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v0, s[2:3], s0, v0
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s[2:3]
+; GFX11-NEXT: s_mov_b64 s[0:1], exec
+; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-NEXT: s_cbranch_execnz .LBB3_1
+; GFX11-NEXT: ; %bb.3: ; %bb
+; GFX11-NEXT: s_getpc_b64 s[2:3]
+; GFX11-NEXT: .Lpost_getpc3:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s2, s2, (.LBB3_2-.Lpost_getpc3)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB3_2-.Lpost_getpc3)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[2:3]
+; GFX11-NEXT: .LBB3_1: ; %bb2
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: ; 32 bytes
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: .LBB3_2: ; %bb3
+; GFX11-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX11-NEXT: global_store_b32 v[0:1], v2, off dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_nop 0
+; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: min_long_forward_vbranch:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
+; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: global_load_b32 v2, v0, s[0:1] scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX12-NEXT: s_cbranch_execnz .LBB3_1
+; GFX12-NEXT: ; %bb.3: ; %bb
+; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: .Lpost_getpc3:
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_sext_i32_i16 s3, s3
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB3_2-.Lpost_getpc3)&4294967295
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB3_2-.Lpost_getpc3)>>32
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_setpc_b64 s[2:3]
+; GFX12-NEXT: .LBB3_1: ; %bb2
+; GFX12-NEXT: ;;#ASMSTART
+; GFX12-NEXT: ; 32 bytes
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: ;;#ASMEND
+; GFX12-NEXT: .LBB3_2: ; %bb3
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_nop 0
+; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT: s_endpgm
bb:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = zext i32 %tid to i64
@@ -237,6 +528,63 @@ define amdgpu_kernel void @long_backward_sbranch(ptr addrspace(1) %arg) #0 {
; GCN-NEXT: s_setpc_b64 s[2:3]
; GCN-NEXT: .LBB4_2: ; %bb3
; GCN-NEXT: s_endpgm
+;
+; GFX11-LABEL: long_backward_sbranch:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB4_1: ; %bb2
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s0, s0, 1
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: s_cmp_lt_i32 s0, 10
+; GFX11-NEXT: s_cbranch_scc0 .LBB4_2
+; GFX11-NEXT: ; %bb.3: ; %bb2
+; GFX11-NEXT: ; in Loop: Header=BB4_1 Depth=1
+; GFX11-NEXT: s_getpc_b64 s[2:3]
+; GFX11-NEXT: .Lpost_getpc4:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s2, s2, (.LBB4_1-.Lpost_getpc4)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB4_1-.Lpost_getpc4)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[2:3]
+; GFX11-NEXT: .LBB4_2: ; %bb3
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: long_backward_sbranch:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: .LBB4_1: ; %bb2
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_add_co_i32 s0, s0, 1
+; GFX12-NEXT: ;;#ASMSTART
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: v_nop_e64
+; GFX12-NEXT: ;;#ASMEND
+; GFX12-NEXT: s_cmp_lt_i32 s0, 10
+; GFX12-NEXT: s_cbranch_scc0 .LBB4_2
+; GFX12-NEXT: ; %bb.3: ; %bb2
+; GFX12-NEXT: ; in Loop: Header=BB4_1 Depth=1
+; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: .Lpost_getpc4:
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_sext_i32_i16 s3, s3
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB4_1-.Lpost_getpc4)&4294967295
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB4_1-.Lpost_getpc4)>>32
+; GFX12-NEXT: s_wait_alu 0xfffe
+; GFX12-NEXT: s_setpc_b64 s[2:3]
+; GFX12-NEXT: .LBB4_2: ; %bb3
+; GFX12-NEXT: s_endpgm
bb:
br label %bb2
@@ -311,6 +659,131 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: s_add_u32 s0, s0, (.LBB5_3-.Lpost_getpc4)&4294967295
; GCN-NEXT: s_addc_u32 s1, s1, (.LBB5_3-.Lpost_getpc4)>>32
; GCN-NEXT: s_setpc_b64 s[0:1]
+;
+; GFX11-LABEL: uniform_unconditional_min_long_forward_branch:
+; GFX11: ; %bb.0: ; %bb0
+; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x2c
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-NEXT: s_cbranch_scc0 .LBB5_1
+; GFX11-NEXT: ; %bb.7: ; %bb0
+; GFX11-NEXT: s_getpc_b64 s[0:1]
+; GFX11-NEXT: .Lpost_getpc6:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s0, s0, (.LBB5_4-.Lpost_getpc6)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s1, s1, (.LBB5_4-.Lpost_getpc6)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[0:1]
+; GFX11-NEXT: .LBB5_1: ; %Flow
+; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[0:1]
+; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
+; GFX11-NEXT: .LBB5_2: ; %bb2
+; GFX11-NEXT: v_mov_b32_e32 v0, 17
+; GFX11-NEXT: global_store_b32 v[0:1], v0, off dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: .LBB5_3: ; %bb4
+; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x24
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, 63
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_nop 0
+; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT: s_endpgm
+; GFX11-NEXT: .LBB5_4: ; %bb3
+; GFX11-NEXT: ;;#ASMSTART
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: v_nop_e64
+; GFX11-NEXT: ;;#ASMEND
+; GFX11-NEXT: s_cbranch_execnz .LBB5_5
+; GFX11-NEXT: ; %bb.9: ; %bb3
+; GFX11-NEXT: s_getpc_b64 s[0:1]
+; GFX11-NEXT: .Lpost_getpc7:
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_add_u32 s0, s0, (.LBB5_2-.Lpost_getpc7)&4294967295
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_addc_u32 s1, s1, (.LBB5_2-.Lpost_getpc7)>>32
+; GFX11-NEXT: s_waitcnt_depctr 0xfffe
+; GFX11-NEXT: s_setpc_b64 s[0:1]
+; GFX11-NEXT: .LBB5_5: ; %bb3
+; GFX11-NEXT: s_getpc_b64 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/109233
More information about the llvm-commits
mailing list