[llvm] [AMDGPU][GCNPreRAOptimizations] Reduce BVH premature reuse (PR #197386)
Carl Ritson via llvm-commits
llvm-commits at lists.llvm.org
Wed May 13 04:02:14 PDT 2026
https://github.com/perlfu updated https://github.com/llvm/llvm-project/pull/197386
>From 19dc628384bb4490d4560b4e421814473b0451c3 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 13 May 2026 17:14:36 +0900
Subject: [PATCH 1/2] [AMDGPU][GCNPreRAOptimizations] Reduce BVH premature
reuse
Add implicit uses to ds_bvh_stack instructions to avoid reuse of
VGPRs allocated to bvh_intersect_ray results prior to ds_bvh_stack.
This reduces likelihood of a premature s_wait_bvhcnt occuring
due to partial reallocation of unused bvh_intersect_ray results
registers.
---
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 100 ++++--
.../AMDGPU/optimize-ds-bvh-stack-pre-ra.ll | 300 ++++++++++++++++++
2 files changed, 377 insertions(+), 23 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/optimize-ds-bvh-stack-pre-ra.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index cd56887fd46a8..1680876140f9b 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -53,6 +53,8 @@ class GCNPreRAOptimizationsImpl {
LiveIntervals *LIS;
bool processReg(Register Reg);
+ void hintTrue16Copy(const MachineInstr &MI);
+ bool optimizeBVHStack(MachineInstr &MI);
public:
GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
@@ -238,6 +240,65 @@ GCNPreRAOptimizationsPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
}
+void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
+ bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
+ if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
+ TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
+ if (Src.isVirtual() && MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
+ Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
+ if (!Dst.isVirtual() || !Src.isVirtual())
+ return;
+ if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
+ MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
+ MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
+ MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
+ }
+ if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
+ MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
+}
+
+bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
+ SmallVector<Register> UseRegs;
+
+ // Find all BVH sources for this DS_BVH_STACK instruction.
+ for (MachineOperand &Use : MI.uses()) {
+ if (!Use.isReg() || Use.isImplicit())
+ continue;
+ Register Reg = Use.getReg();
+ for (const MachineInstr &Src : MRI->def_instructions(Reg)) {
+ if (!SIInstrInfo::isImage(Src))
+ continue;
+ const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Src.getOpcode());
+ const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
+ AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
+ if (!BaseInfo->BVH)
+ continue;
+ UseRegs.push_back(Reg);
+ break;
+ }
+ }
+
+ if (UseRegs.empty())
+ return false;
+
+ // Add implicit uses for entire BVH source registers.
+ // This avoids partial reallocation of register which could
+ // introduce a premature s_wait_bvhcnt.
+ for (Register Reg : UseRegs) {
+ MI.addOperand(MachineOperand::CreateReg(Reg, false, true));
+ LIS->removeInterval(Reg);
+ LIS->createAndComputeVirtRegInterval(Reg);
+ }
+ LLVM_DEBUG(dbgs() << "Added implicit uses to: " << MI);
+
+ return true;
+}
+
bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
@@ -258,34 +319,27 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
Changed |= processReg(Reg);
}
- if (!ST.useRealTrue16Insts())
+ const bool HasBVHStack = ST.hasImageInsts() && ST.hasBVHDualAndBVH8Insts();
+ const bool HasRealTrue16 = ST.useRealTrue16Insts();
+
+ if (!HasRealTrue16 && !HasBVHStack)
return Changed;
- // Add RA hints to improve True16 COPY elimination.
- for (const MachineBasicBlock &MBB : MF) {
- for (const MachineInstr &MI : MBB) {
- if (MI.getOpcode() != AMDGPU::COPY)
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ // Add RA hints to improve True16 COPY elimination.
+ if (HasRealTrue16 && MI.getOpcode() == AMDGPU::COPY) {
+ hintTrue16Copy(MI);
continue;
- Register Dst = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(1).getReg();
- const TargetRegisterClass *DstRC = TRI->getRegClassForReg(*MRI, Dst);
- bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(DstRC);
- if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
- TRI->getRegClassForReg(*MRI, Src) == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Dst, 0, TRI->getSubReg(Src, AMDGPU::lo16));
- if (Src.isVirtual() &&
- MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass &&
- Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Src, 0, TRI->getSubReg(Dst, AMDGPU::lo16));
- if (!Dst.isVirtual() || !Src.isVirtual())
+ }
+ // Add implicit uses to avoid early wait on intersect ray instructions.
+ if (HasBVHStack &&
+ (MI.getOpcode() == AMDGPU::DS_BVH_STACK_RTN_B32 ||
+ MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32 ||
+ MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64)) {
+ Changed |= optimizeBVHStack(MI);
continue;
- if (MRI->getRegClass(Dst) == &AMDGPU::VGPR_32RegClass &&
- MRI->getRegClass(Src) == &AMDGPU::VGPR_16RegClass) {
- MRI->setRegAllocationHint(Dst, AMDGPURI::Size32, Src);
- MRI->setRegAllocationHint(Src, AMDGPURI::Size16, Dst);
}
- if (IsDst16Bit && MRI->getRegClass(Src) == &AMDGPU::VGPR_32RegClass)
- MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-ds-bvh-stack-pre-ra.ll b/llvm/test/CodeGen/AMDGPU/optimize-ds-bvh-stack-pre-ra.ll
new file mode 100644
index 0000000000000..be351ea026a03
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/optimize-ds-bvh-stack-pre-ra.ll
@@ -0,0 +1,300 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
+
+define amdgpu_gs void @test_ds_bvh_stack_push4_pop1(i32 %addr, i32 %data.0, i64 %node_ptr, float %ray_extent, float %ray_origin_x, float %ray_origin_y, float %ray_origin_z, float %ray_dir_x, float %ray_dir_y, float %ray_dir_z, i32 %offset, <4 x i32> inreg %tdescr, ptr addrspace(1) %p.0, ptr addrspace(1) %p.1, ptr addrspace(1) %p.2, ptr addrspace(1) %p.3) {
+; CHECK-LABEL: test_ds_bvh_stack_push4_pop1:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v21, v6
+; CHECK-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v5, 0
+; CHECK-NEXT: s_mov_b32 s4, exec_lo
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_dual_mov_b32 v33, v22 :: v_dual_mov_b32 v32, v21
+; CHECK-NEXT: v_mov_b32_e32 v31, v20
+; CHECK-NEXT: image_bvh8_intersect_ray v[21:30], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: v_cmpx_eq_f32_e32 0, v20
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: ; %bb.1: ; %if
+; CHECK-NEXT: global_load_b64 v[6:7], v[12:13], off
+; CHECK-NEXT: global_load_b64 v[34:35], v[14:15], off
+; CHECK-NEXT: global_load_b64 v[36:37], v[16:17], off
+; CHECK-NEXT: global_load_b64 v[38:39], v[18:19], off
+; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: v_add_nc_u32_e32 v1, v7, v35
+; CHECK-NEXT: s_wait_loadcnt 0x1
+; CHECK-NEXT: v_add3_u32 v6, v6, v34, v36
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT: v_add3_u32 v1, v1, v37, v39
+; CHECK-NEXT: v_add3_u32 v1, v6, v38, v1
+; CHECK-NEXT: .LBB0_2: ; %end
+; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: ds_bvh_stack_push4_pop1_rtn_b32 v1, v0, v1, v[21:24]
+; CHECK-NEXT: image_bvh8_intersect_ray v[20:29], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: s_wait_dscnt 0x0
+; CHECK-NEXT: global_store_b32 v[12:13], v1, off
+; CHECK-NEXT: global_store_b32 v[14:15], v0, off
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: global_store_b32 v[16:17], v20, off
+; CHECK-NEXT: global_store_b32 v[18:19], v21, off
+; CHECK-NEXT: s_endpgm
+entry:
+ %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
+ %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
+ %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
+ %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
+ %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
+ %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
+ %v = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %ray_origin, <3 x float> %ray_dir, i32 %offset, <4 x i32> %tdescr)
+ %a = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 0
+ %val.0 = extractelement <10 x i32> %a, i32 0
+ %val.1 = extractelement <10 x i32> %a, i32 1
+ %val.2 = extractelement <10 x i32> %a, i32 2
+ %val.3 = extractelement <10 x i32> %a, i32 3
+ %bvh.0 = insertelement <4 x i32> poison, i32 %val.0, i32 0
+ %bvh.1 = insertelement <4 x i32> %bvh.0, i32 %val.1, i32 1
+ %bvh.2 = insertelement <4 x i32> %bvh.1, i32 %val.2, i32 2
+ %bvh = insertelement <4 x i32> %bvh.2, i32 %val.3, i32 3
+ %cnd = fcmp oeq float %ray_origin_x, 0.0
+ br i1 %cnd, label %if, label %end
+
+if:
+ ; loads to force vgpr pressure
+ %load.0 = load <2 x i32>, ptr addrspace(1) %p.0
+ %load.1 = load <2 x i32>, ptr addrspace(1) %p.1
+ %load.2 = load <2 x i32>, ptr addrspace(1) %p.2
+ %load.3 = load <2 x i32>, ptr addrspace(1) %p.3
+ %add.0 = add <2 x i32> %load.0, %load.1
+ %add.1 = add <2 x i32> %add.0, %load.2
+ %add.2 = add <2 x i32> %add.1, %load.3
+ %.i0 = extractelement <2 x i32> %add.2, i32 0
+ %.i1 = extractelement <2 x i32> %add.2, i32 1
+ %data.1 = add i32 %.i0, %.i1
+ br label %end
+
+end:
+ %data = phi i32 [ %data.0, %entry ], [ %data.1, %if ]
+ %pair = call { i32, i32 } @llvm.amdgcn.ds.bvh.stack.push4.pop1.rtn(i32 %addr, i32 %data, <4 x i32> %bvh, i32 0)
+ %vdst = extractvalue { i32, i32 } %pair, 0
+ %newaddr = extractvalue { i32, i32 } %pair, 1
+
+ ; keep all intersect ray parameters live
+ %new.origin = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 1
+ %new.dir = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 2
+ %v.2 = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %new.origin, <3 x float> %new.dir, i32 %offset, <4 x i32> %tdescr)
+ %b = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v.2, 0
+ %c = extractelement <10 x i32> %b, i32 0
+ %d = extractelement <10 x i32> %b, i32 1
+
+ ; stores keep pointers live
+ store i32 %vdst, ptr addrspace(1) %p.0
+ store i32 %newaddr, ptr addrspace(1) %p.1
+ store i32 %c, ptr addrspace(1) %p.2
+ store i32 %d, ptr addrspace(1) %p.3
+
+ ret void
+}
+
+define amdgpu_gs void @test_ds_bvh_stack_push8_pop1(i32 %addr, i32 %data.0, i64 %node_ptr, float %ray_extent, float %ray_origin_x, float %ray_origin_y, float %ray_origin_z, float %ray_dir_x, float %ray_dir_y, float %ray_dir_z, i32 %offset, <4 x i32> inreg %tdescr, ptr addrspace(1) %p.0, ptr addrspace(1) %p.1, ptr addrspace(1) %p.2, ptr addrspace(1) %p.3) {
+; CHECK-LABEL: test_ds_bvh_stack_push8_pop1:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v21, v6
+; CHECK-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v5, 0
+; CHECK-NEXT: s_mov_b32 s4, exec_lo
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_dual_mov_b32 v33, v22 :: v_dual_mov_b32 v32, v21
+; CHECK-NEXT: v_mov_b32_e32 v31, v20
+; CHECK-NEXT: image_bvh8_intersect_ray v[21:30], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: v_cmpx_eq_f32_e32 0, v20
+; CHECK-NEXT: s_cbranch_execz .LBB1_2
+; CHECK-NEXT: ; %bb.1: ; %if
+; CHECK-NEXT: global_load_b64 v[6:7], v[12:13], off
+; CHECK-NEXT: global_load_b64 v[34:35], v[14:15], off
+; CHECK-NEXT: global_load_b64 v[36:37], v[16:17], off
+; CHECK-NEXT: global_load_b64 v[38:39], v[18:19], off
+; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: v_add_nc_u32_e32 v1, v7, v35
+; CHECK-NEXT: s_wait_loadcnt 0x1
+; CHECK-NEXT: v_add3_u32 v6, v6, v34, v36
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT: v_add3_u32 v1, v1, v37, v39
+; CHECK-NEXT: v_add3_u32 v1, v6, v38, v1
+; CHECK-NEXT: .LBB1_2: ; %end
+; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: ds_bvh_stack_push8_pop1_rtn_b32 v1, v0, v1, v[21:28]
+; CHECK-NEXT: image_bvh8_intersect_ray v[20:29], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: s_wait_dscnt 0x0
+; CHECK-NEXT: global_store_b32 v[12:13], v1, off
+; CHECK-NEXT: global_store_b32 v[14:15], v0, off
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: global_store_b32 v[16:17], v20, off
+; CHECK-NEXT: global_store_b32 v[18:19], v21, off
+; CHECK-NEXT: s_endpgm
+entry:
+ %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
+ %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
+ %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
+ %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
+ %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
+ %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
+ %v = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %ray_origin, <3 x float> %ray_dir, i32 %offset, <4 x i32> %tdescr)
+ %a = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 0
+ %val.0 = extractelement <10 x i32> %a, i32 0
+ %val.1 = extractelement <10 x i32> %a, i32 1
+ %val.2 = extractelement <10 x i32> %a, i32 2
+ %val.3 = extractelement <10 x i32> %a, i32 3
+ %val.4 = extractelement <10 x i32> %a, i32 4
+ %val.5 = extractelement <10 x i32> %a, i32 5
+ %val.6 = extractelement <10 x i32> %a, i32 6
+ %val.7 = extractelement <10 x i32> %a, i32 7
+ %bvh.0 = insertelement <8 x i32> poison, i32 %val.0, i32 0
+ %bvh.1 = insertelement <8 x i32> %bvh.0, i32 %val.1, i32 1
+ %bvh.2 = insertelement <8 x i32> %bvh.1, i32 %val.2, i32 2
+ %bvh.3 = insertelement <8 x i32> %bvh.2, i32 %val.3, i32 3
+ %bvh.4 = insertelement <8 x i32> %bvh.3, i32 %val.4, i32 4
+ %bvh.5 = insertelement <8 x i32> %bvh.4, i32 %val.5, i32 5
+ %bvh.6 = insertelement <8 x i32> %bvh.5, i32 %val.6, i32 6
+ %bvh = insertelement <8 x i32> %bvh.6, i32 %val.7, i32 7
+ %cnd = fcmp oeq float %ray_origin_x, 0.0
+ br i1 %cnd, label %if, label %end
+
+if:
+ ; loads to force vgpr pressure
+ %load.0 = load <2 x i32>, ptr addrspace(1) %p.0
+ %load.1 = load <2 x i32>, ptr addrspace(1) %p.1
+ %load.2 = load <2 x i32>, ptr addrspace(1) %p.2
+ %load.3 = load <2 x i32>, ptr addrspace(1) %p.3
+ %add.0 = add <2 x i32> %load.0, %load.1
+ %add.1 = add <2 x i32> %add.0, %load.2
+ %add.2 = add <2 x i32> %add.1, %load.3
+ %.i0 = extractelement <2 x i32> %add.2, i32 0
+ %.i1 = extractelement <2 x i32> %add.2, i32 1
+ %data.1 = add i32 %.i0, %.i1
+ br label %end
+
+end:
+ %data = phi i32 [ %data.0, %entry ], [ %data.1, %if ]
+ %pair = call { i32, i32 } @llvm.amdgcn.ds.bvh.stack.push8.pop1.rtn(i32 %addr, i32 %data, <8 x i32> %bvh, i32 0)
+ %vdst = extractvalue { i32, i32 } %pair, 0
+ %newaddr = extractvalue { i32, i32 } %pair, 1
+
+ ; keep all intersect ray parameters live
+ %new.origin = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 1
+ %new.dir = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 2
+ %v.2 = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %new.origin, <3 x float> %new.dir, i32 %offset, <4 x i32> %tdescr)
+ %b = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v.2, 0
+ %c = extractelement <10 x i32> %b, i32 0
+ %d = extractelement <10 x i32> %b, i32 1
+
+ ; stores keep pointers live
+ store i32 %vdst, ptr addrspace(1) %p.0
+ store i32 %newaddr, ptr addrspace(1) %p.1
+ store i32 %c, ptr addrspace(1) %p.2
+ store i32 %d, ptr addrspace(1) %p.3
+
+ ret void
+}
+
+define amdgpu_gs void @test_ds_bvh_stack_push8_pop2(i32 %addr, i32 %data.0, i64 %node_ptr, float %ray_extent, float %ray_origin_x, float %ray_origin_y, float %ray_origin_z, float %ray_dir_x, float %ray_dir_y, float %ray_dir_z, i32 %offset, <4 x i32> inreg %tdescr, ptr addrspace(1) %p.0, ptr addrspace(1) %p.1, ptr addrspace(1) %p.2, ptr addrspace(1) %p.3) {
+; CHECK-LABEL: test_ds_bvh_stack_push8_pop2:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: v_dual_mov_b32 v22, v7 :: v_dual_mov_b32 v21, v6
+; CHECK-NEXT: v_dual_mov_b32 v20, v5 :: v_dual_mov_b32 v5, 0
+; CHECK-NEXT: s_mov_b32 s4, exec_lo
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_dual_mov_b32 v33, v22 :: v_dual_mov_b32 v32, v21
+; CHECK-NEXT: v_mov_b32_e32 v31, v20
+; CHECK-NEXT: image_bvh8_intersect_ray v[21:30], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: v_cmpx_eq_f32_e32 0, v20
+; CHECK-NEXT: s_cbranch_execz .LBB2_2
+; CHECK-NEXT: ; %bb.1: ; %if
+; CHECK-NEXT: global_load_b64 v[6:7], v[12:13], off
+; CHECK-NEXT: global_load_b64 v[34:35], v[14:15], off
+; CHECK-NEXT: global_load_b64 v[36:37], v[16:17], off
+; CHECK-NEXT: global_load_b64 v[38:39], v[18:19], off
+; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: v_add_nc_u32_e32 v1, v7, v35
+; CHECK-NEXT: s_wait_loadcnt 0x1
+; CHECK-NEXT: v_add3_u32 v6, v6, v34, v36
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; CHECK-NEXT: v_add3_u32 v1, v1, v37, v39
+; CHECK-NEXT: v_add3_u32 v1, v6, v38, v1
+; CHECK-NEXT: .LBB2_2: ; %end
+; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: ds_bvh_stack_push8_pop2_rtn_b64 v[6:7], v0, v1, v[21:28]
+; CHECK-NEXT: image_bvh8_intersect_ray v[20:29], [v[2:3], v[4:5], v[31:33], v[8:10], v11], s[0:3]
+; CHECK-NEXT: s_wait_dscnt 0x0
+; CHECK-NEXT: global_store_b64 v[12:13], v[6:7], off
+; CHECK-NEXT: global_store_b32 v[14:15], v0, off
+; CHECK-NEXT: s_wait_bvhcnt 0x0
+; CHECK-NEXT: global_store_b32 v[16:17], v20, off
+; CHECK-NEXT: global_store_b32 v[18:19], v21, off
+; CHECK-NEXT: s_endpgm
+entry:
+ %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
+ %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
+ %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
+ %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
+ %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
+ %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
+ %v = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %ray_origin, <3 x float> %ray_dir, i32 %offset, <4 x i32> %tdescr)
+ %a = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 0
+ %val.0 = extractelement <10 x i32> %a, i32 0
+ %val.1 = extractelement <10 x i32> %a, i32 1
+ %val.2 = extractelement <10 x i32> %a, i32 2
+ %val.3 = extractelement <10 x i32> %a, i32 3
+ %val.4 = extractelement <10 x i32> %a, i32 4
+ %val.5 = extractelement <10 x i32> %a, i32 5
+ %val.6 = extractelement <10 x i32> %a, i32 6
+ %val.7 = extractelement <10 x i32> %a, i32 7
+ %bvh.0 = insertelement <8 x i32> poison, i32 %val.0, i32 0
+ %bvh.1 = insertelement <8 x i32> %bvh.0, i32 %val.1, i32 1
+ %bvh.2 = insertelement <8 x i32> %bvh.1, i32 %val.2, i32 2
+ %bvh.3 = insertelement <8 x i32> %bvh.2, i32 %val.3, i32 3
+ %bvh.4 = insertelement <8 x i32> %bvh.3, i32 %val.4, i32 4
+ %bvh.5 = insertelement <8 x i32> %bvh.4, i32 %val.5, i32 5
+ %bvh.6 = insertelement <8 x i32> %bvh.5, i32 %val.6, i32 6
+ %bvh = insertelement <8 x i32> %bvh.6, i32 %val.7, i32 7
+ %cnd = fcmp oeq float %ray_origin_x, 0.0
+ br i1 %cnd, label %if, label %end
+
+if:
+ ; loads to force vgpr pressure
+ %load.0 = load <2 x i32>, ptr addrspace(1) %p.0
+ %load.1 = load <2 x i32>, ptr addrspace(1) %p.1
+ %load.2 = load <2 x i32>, ptr addrspace(1) %p.2
+ %load.3 = load <2 x i32>, ptr addrspace(1) %p.3
+ %add.0 = add <2 x i32> %load.0, %load.1
+ %add.1 = add <2 x i32> %add.0, %load.2
+ %add.2 = add <2 x i32> %add.1, %load.3
+ %.i0 = extractelement <2 x i32> %add.2, i32 0
+ %.i1 = extractelement <2 x i32> %add.2, i32 1
+ %data.1 = add i32 %.i0, %.i1
+ br label %end
+
+end:
+ %data = phi i32 [ %data.0, %entry ], [ %data.1, %if ]
+ %pair = call { i64, i32 } @llvm.amdgcn.ds.bvh.stack.push8.pop2.rtn(i32 %addr, i32 %data, <8 x i32> %bvh, i32 0)
+ %vdst = extractvalue { i64, i32 } %pair, 0
+ %newaddr = extractvalue { i64, i32 } %pair, 1
+
+ ; keep all intersect ray parameters live
+ %new.origin = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 1
+ %new.dir = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v, 2
+ %v.2 = call {<10 x i32>, <3 x float>, <3 x float>} @llvm.amdgcn.image.bvh8.intersect.ray(i64 %node_ptr, float %ray_extent, i8 0, <3 x float> %new.origin, <3 x float> %new.dir, i32 %offset, <4 x i32> %tdescr)
+ %b = extractvalue {<10 x i32>, <3 x float>, <3 x float>} %v.2, 0
+ %c = extractelement <10 x i32> %b, i32 0
+ %d = extractelement <10 x i32> %b, i32 1
+
+ ; stores keep pointers live
+ store i64 %vdst, ptr addrspace(1) %p.0
+ store i32 %newaddr, ptr addrspace(1) %p.1
+ store i32 %c, ptr addrspace(1) %p.2
+ store i32 %d, ptr addrspace(1) %p.3
+
+ ret void
+}
>From 8e9689e6d1a015aab2f426f6b5cd0573f4179c84 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 13 May 2026 20:01:31 +0900
Subject: [PATCH 2/2] - Integrate reviewer suggestions
---
llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index 1680876140f9b..acc01aadd7a91 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -263,12 +263,10 @@ void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
}
bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
- SmallVector<Register> UseRegs;
+ SmallVector<Register, 2> UseRegs;
- // Find all BVH sources for this DS_BVH_STACK instruction.
- for (MachineOperand &Use : MI.uses()) {
- if (!Use.isReg() || Use.isImplicit())
- continue;
+ // Find BVH sources for this DS_BVH_STACK instruction.
+ auto CheckUse = [&](MachineOperand &Use) {
Register Reg = Use.getReg();
for (const MachineInstr &Src : MRI->def_instructions(Reg)) {
if (!SIInstrInfo::isImage(Src))
@@ -281,7 +279,9 @@ bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
UseRegs.push_back(Reg);
break;
}
- }
+ };
+ CheckUse(*TII->getNamedOperand(MI, AMDGPU::OpName::data0));
+ CheckUse(*TII->getNamedOperand(MI, AMDGPU::OpName::data1));
if (UseRegs.empty())
return false;
More information about the llvm-commits
mailing list