[llvm] [AMDGPU] Fix waterfall inreg call args -- AGPR sources legalized for `V_READFIRSTLANE_B32` instr (PR #194890)
Amit Tiwari via llvm-commits
llvm-commits at lists.llvm.org
Thu May 14 00:02:40 PDT 2026
https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/194890
>From d0fc4c89bc5c8fbeea483692e2fde60e372cbe64 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 29 Apr 2026 11:11:36 -0400
Subject: [PATCH 1/6] waterfall-inreg-fix
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 47 +++++++++++++++++++
.../AMDGPU/waterfall-call-inreg-agpr.ll | 47 +++++++++++++++++++
2 files changed, 94 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0c0d258bf8f1b..2aa70ff389315 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7109,6 +7109,15 @@ static void emitLoadScalarOpsFromVGPRLoop(
unsigned NumSubRegs = RegSize / 32;
Register VScalarOp = ScalarOp->getReg();
+ // V_READFIRSTLANE_B32 / V_CMP_EQ_U32_e64 reject AGPR sources, so demote
+ // AV_* operands into a pure VGPR class on subtargets with AGPRs.
+ if (TRI->hasAGPRs(MRI.getRegClass(VScalarOp))) {
+ Register VRReg = MRI.createVirtualRegister(
+ TRI->getEquivalentVGPRClass(MRI.getRegClass(VScalarOp)));
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
+ VScalarOp = VRReg;
+ }
+
if (NumSubRegs == 1) {
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
@@ -7828,6 +7837,44 @@ void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
MI->definesRegister(End->getOperand(1).getReg(), &RI))
++End;
+ // generateWaterFallLoop() splices [Start, End) into BodyBB, but the
+ // readfirstlane/compare it emits live in LoopBB and execute before
+ // BodyBB on the first iteration. Hoist simple-COPY defs of ScalarOp
+ // registers whose source already dominates Start so they stay in MBB
+ // (LoopBB's predecessor) and the readfirstlane sees a defined value.
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+ auto IsBeforeStart = [&](const MachineInstr *Q) {
+ if (!Q)
+ return false;
+ if (Q->getParent() != &MBB)
+ return true;
+ for (auto It = MBB.begin(); It != Start; ++It)
+ if (&*It == Q)
+ return true;
+ return false;
+ };
+ for (MachineOperand *MO : ScalarOps) {
+ if (!MO->isReg())
+ continue;
+ Register R = MO->getReg();
+ if (!R.isVirtual() || !MRI.hasOneDef(R))
+ continue;
+ MachineInstr *DefMI = MRI.getVRegDef(R);
+ if (!DefMI || !DefMI->isCopy() || DefMI->getParent() != &MBB ||
+ IsBeforeStart(DefMI))
+ continue;
+ const MachineOperand &Src = DefMI->getOperand(1);
+ if (!Src.isReg() || !Src.getReg().isVirtual() ||
+ !IsBeforeStart(MRI.getVRegDef(Src.getReg())))
+ continue;
+ // The COPY moves out of [Begin, MI], so generateWaterFallLoop's kill-flag
+ // sweep no longer covers it; clear them here to keep any remaining use of
+ // Src inside BodyBB valid.
+ for (MachineOperand &U : DefMI->all_uses())
+ MRI.clearKillFlags(U.getReg());
+ MBB.splice(Start, &MBB, DefMI->getIterator());
+ }
+
generateWaterFallLoop(*this, *MI, ScalarOps, MDT, Start, End, PhySGPRs);
}
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
new file mode 100644
index 0000000000000..1211c3bb34976
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -O3 -verify-machineinstrs < %s | FileCheck %s
+
+; On gfx90a, an inreg call argument can be allocated to an AV_* register
+; class (VGPR-or-AGPR). The waterfall loop in SIInstrInfo's
+; emitLoadScalarOpsFromVGPRLoop must not feed an AGPR source directly into
+; V_READFIRSTLANE_B32 / V_CMP_EQ_U32_e64; doing so trips the verifier with
+; "Operand has incorrect register class" (`$sgprN = V_READFIRSTLANE_B32
+; $agpr0`) and aborts in LiveRangeCalc / MachineVerifier. The fix demotes
+; the operand into a pure VGPR class via a COPY before reading it in the
+; loop.
+
+ at G = global ptr poison
+ at G.1 = global ptr poison
+ at G.2 = global ptr addrspace(1) poison
+ at G.3 = global ptr addrspace(3) poison
+ at G.4 = global ptr addrspace(5) poison
+ at G.5 = global ptr addrspace(5) poison
+
+declare amdgpu_cs_chain_preserve void @callee(ptr inreg, ptr,
+ ptr addrspace(1) inreg,
+ ptr addrspace(1),
+ ptr addrspace(3) inreg,
+ ptr addrspace(3),
+ ptr addrspace(5) inreg,
+ ptr addrspace(5))
+
+; The call must waterfall the inreg operands. The source of every emitted
+; v_readfirstlane_b32 in this loop must be a VGPR (v#), never an AGPR (a#);
+; otherwise we hit "Operand has incorrect register class" on gfx90a.
+;
+; CHECK-LABEL: chain_preserve_caller:
+; CHECK-NOT: v_readfirstlane_b32 s{{[0-9]+}}, a{{[0-9]+}}
+define amdgpu_cs_chain_preserve void @chain_preserve_caller(float inreg %a, float %b) {
+ %LGV6 = load ptr addrspace(5), ptr @G.5, align 8
+ %LGV5 = load ptr addrspace(5), ptr @G.4, align 8
+ %LGV4 = load ptr addrspace(3), ptr @G.3, align 8
+ %LGV2 = load ptr addrspace(1), ptr @G.2, align 8
+ %LGV1 = load ptr, ptr @G.1, align 8
+ %LGV = load ptr, ptr @G, align 8
+ %c = fadd float %a, %b
+ store float %c, ptr poison, align 4
+ call void @callee(ptr %LGV, ptr %LGV1,
+ ptr addrspace(1) %LGV2, ptr addrspace(1) poison,
+ ptr addrspace(3) poison, ptr addrspace(3) %LGV4,
+ ptr addrspace(5) %LGV5, ptr addrspace(5) %LGV6)
+ ret void
+}
>From 37678ee79a9e97ca8b023552ef92530a7221ae4d Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 30 Apr 2026 07:33:51 -0400
Subject: [PATCH 2/6] revised
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 42 +++---
.../AMDGPU/waterfall-call-inreg-agpr.ll | 120 +++++++++++++++---
2 files changed, 130 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2aa70ff389315..cffad71c8c291 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7109,11 +7109,25 @@ static void emitLoadScalarOpsFromVGPRLoop(
unsigned NumSubRegs = RegSize / 32;
Register VScalarOp = ScalarOp->getReg();
- // V_READFIRSTLANE_B32 / V_CMP_EQ_U32_e64 reject AGPR sources, so demote
- // AV_* operands into a pure VGPR class on subtargets with AGPRs.
- if (TRI->hasAGPRs(MRI.getRegClass(VScalarOp))) {
- Register VRReg = MRI.createVirtualRegister(
- TRI->getEquivalentVGPRClass(MRI.getRegClass(VScalarOp)));
+ auto LaneClassOf = [&](const TargetRegisterClass *RC) {
+ if (TRI->getRegSizeInBits(*RC) <= 32)
+ return RC;
+ const TargetRegisterClass *Sub =
+ TRI->getSubRegisterClass(RC, AMDGPU::sub0);
+ return Sub ? Sub : RC;
+ };
+ const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(VScalarOp);
+ const TargetRegisterClass *RFLSrcRC = TRI->getRegClass(TII.getOpRegClassID(
+ TII.get(AMDGPU::V_READFIRSTLANE_B32).operands()[1]));
+ const TargetRegisterClass *LaneRC = LaneClassOf(VScalarOpRC);
+ if (RFLSrcRC && TRI->getCommonSubClass(LaneRC, RFLSrcRC) != LaneRC) {
+ const TargetRegisterClass *DemoteRC =
+ TRI->getEquivalentVGPRClass(VScalarOpRC);
+ assert(DemoteRC &&
+ TRI->getCommonSubClass(LaneClassOf(DemoteRC), RFLSrcRC) ==
+ LaneClassOf(DemoteRC) &&
+ "demote target lane class incompatible with V_READFIRSTLANE_B32");
+ Register VRReg = MRI.createVirtualRegister(DemoteRC);
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
VScalarOp = VRReg;
}
@@ -7837,15 +7851,12 @@ void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
MI->definesRegister(End->getOperand(1).getReg(), &RI))
++End;
- // generateWaterFallLoop() splices [Start, End) into BodyBB, but the
- // readfirstlane/compare it emits live in LoopBB and execute before
- // BodyBB on the first iteration. Hoist simple-COPY defs of ScalarOp
- // registers whose source already dominates Start so they stay in MBB
- // (LoopBB's predecessor) and the readfirstlane sees a defined value.
MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
- auto IsBeforeStart = [&](const MachineInstr *Q) {
+ auto DominatesStart = [&](const MachineInstr *Q) -> bool {
if (!Q)
return false;
+ if (MDT)
+ return MDT->dominates(Q, &*Start);
if (Q->getParent() != &MBB)
return true;
for (auto It = MBB.begin(); It != Start; ++It)
@@ -7857,19 +7868,16 @@ void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
if (!MO->isReg())
continue;
Register R = MO->getReg();
- if (!R.isVirtual() || !MRI.hasOneDef(R))
+ if (!R.isVirtual())
continue;
MachineInstr *DefMI = MRI.getVRegDef(R);
if (!DefMI || !DefMI->isCopy() || DefMI->getParent() != &MBB ||
- IsBeforeStart(DefMI))
+ DominatesStart(DefMI))
continue;
const MachineOperand &Src = DefMI->getOperand(1);
if (!Src.isReg() || !Src.getReg().isVirtual() ||
- !IsBeforeStart(MRI.getVRegDef(Src.getReg())))
+ !DominatesStart(MRI.getVRegDef(Src.getReg())))
continue;
- // The COPY moves out of [Begin, MI], so generateWaterFallLoop's kill-flag
- // sweep no longer covers it; clear them here to keep any remaining use of
- // Src inside BodyBB valid.
for (MachineOperand &U : DefMI->all_uses())
MRI.clearKillFlags(U.getReg());
MBB.splice(Start, &MBB, DefMI->getIterator());
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
index 1211c3bb34976..39ac9616ee31b 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
@@ -1,13 +1,9 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -O3 -verify-machineinstrs < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s
-; On gfx90a, an inreg call argument can be allocated to an AV_* register
-; class (VGPR-or-AGPR). The waterfall loop in SIInstrInfo's
-; emitLoadScalarOpsFromVGPRLoop must not feed an AGPR source directly into
-; V_READFIRSTLANE_B32 / V_CMP_EQ_U32_e64; doing so trips the verifier with
-; "Operand has incorrect register class" (`$sgprN = V_READFIRSTLANE_B32
-; $agpr0`) and aborts in LiveRangeCalc / MachineVerifier. The fix demotes
-; the operand into a pure VGPR class via a COPY before reading it in the
-; loop.
+; gfx90a: AV AGPR sources must be copied to VGPR before V_READFIRSTLANE /
+; V_CMP_EQ in the waterfall; defs moved into BodyBB must be hoisted so LoopBB
+; sees them.
@G = global ptr poison
@G.1 = global ptr poison
@@ -24,13 +20,107 @@ declare amdgpu_cs_chain_preserve void @callee(ptr inreg, ptr,
ptr addrspace(5) inreg,
ptr addrspace(5))
-; The call must waterfall the inreg operands. The source of every emitted
-; v_readfirstlane_b32 in this loop must be a VGPR (v#), never an AGPR (a#);
-; otherwise we hit "Operand has incorrect register class" on gfx90a.
-;
-; CHECK-LABEL: chain_preserve_caller:
-; CHECK-NOT: v_readfirstlane_b32 s{{[0-9]+}}, a{{[0-9]+}}
define amdgpu_cs_chain_preserve void @chain_preserve_caller(float inreg %a, float %b) {
+; CHECK-LABEL: chain_preserve_caller:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_getpc_b64 s[2:3]
+; CHECK-NEXT: s_add_u32 s2, s2, G.5 at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s3, s3, G.5 at gotpcrel32@hi+12
+; CHECK-NEXT: s_mov_b64 s[52:53], s[4:5]
+; CHECK-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; CHECK-NEXT: s_getpc_b64 s[4:5]
+; CHECK-NEXT: s_add_u32 s4, s4, G.4 at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s5, s5, G.4 at gotpcrel32@hi+12
+; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
+; CHECK-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; CHECK-NEXT: s_getpc_b64 s[6:7]
+; CHECK-NEXT: s_add_u32 s6, s6, G.3 at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s7, s7, G.3 at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT: s_getpc_b64 s[2:3]
+; CHECK-NEXT: s_add_u32 s2, s2, G.2 at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s3, s3, G.2 at gotpcrel32@hi+12
+; CHECK-NEXT: flat_load_dword a34, v[0:1]
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
+; CHECK-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; CHECK-NEXT: s_getpc_b64 s[4:5]
+; CHECK-NEXT: s_add_u32 s4, s4, G.1 at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s5, s5, G.1 at gotpcrel32@hi+12
+; CHECK-NEXT: flat_load_dword v2, v[0:1]
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
+; CHECK-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; CHECK-NEXT: s_getpc_b64 s[6:7]
+; CHECK-NEXT: s_add_u32 s6, s6, G at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s7, s7, G at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
+; CHECK-NEXT: flat_load_dword a35, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
+; CHECK-NEXT: flat_load_dwordx2 a[32:33], v[0:1]
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
+; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; CHECK-NEXT: v_add_f32_e32 v3, s0, v8
+; CHECK-NEXT: s_mov_b32 s54, s15
+; CHECK-NEXT: s_mov_b32 s55, s14
+; CHECK-NEXT: s_mov_b32 s64, s13
+; CHECK-NEXT: s_mov_b32 s65, s12
+; CHECK-NEXT: s_mov_b64 s[34:35], s[10:11]
+; CHECK-NEXT: s_mov_b64 s[36:37], s[8:9]
+; CHECK-NEXT: s_mov_b64 s[0:1], exec
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: flat_store_dword v[0:1], v3
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: v_readfirstlane_b32 s17, v2
+; CHECK-NEXT: v_readfirstlane_b32 s3, v5
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s17, v2
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v5
+; CHECK-NEXT: v_readfirstlane_b32 s2, v4
+; CHECK-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s2, v4
+; CHECK-NEXT: s_and_b64 s[4:5], s[0:1], vcc
+; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s1, v1
+; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
+; CHECK-NEXT: s_and_saveexec_b64 s[66:67], s[4:5]
+; CHECK-NEXT: s_getpc_b64 s[4:5]
+; CHECK-NEXT: s_add_u32 s4, s4, callee at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s5, s5, callee at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[18:19], s[4:5], 0x0
+; CHECK-NEXT: v_readfirstlane_b32 s16, v0
+; CHECK-NEXT: s_mov_b64 s[4:5], s[52:53]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39]
+; CHECK-NEXT: s_mov_b64 s[8:9], s[36:37]
+; CHECK-NEXT: s_mov_b64 s[10:11], s[34:35]
+; CHECK-NEXT: s_mov_b32 s12, s65
+; CHECK-NEXT: s_mov_b32 s13, s64
+; CHECK-NEXT: s_mov_b32 s14, s55
+; CHECK-NEXT: s_mov_b32 s15, s54
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a35
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a34
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
+; CHECK-NEXT: ; implicit-def: $vgpr31
+; CHECK-NEXT: ; implicit-def: $vgpr3
+; CHECK-NEXT: s_xor_b64 exec, exec, s[66:67]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_1
+; CHECK-NEXT: ; %bb.2:
+; CHECK-NEXT: s_endpgm
%LGV6 = load ptr addrspace(5), ptr @G.5, align 8
%LGV5 = load ptr addrspace(5), ptr @G.4, align 8
%LGV4 = load ptr addrspace(3), ptr @G.3, align 8
>From 9f3db979e6b3a54af92dbe200460cca18569203b Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 1 May 2026 15:32:59 -0400
Subject: [PATCH 3/6] revised
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 32 ++++++++------------------
1 file changed, 10 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index cffad71c8c291..d2fb4e3bee71c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7109,30 +7109,18 @@ static void emitLoadScalarOpsFromVGPRLoop(
unsigned NumSubRegs = RegSize / 32;
Register VScalarOp = ScalarOp->getReg();
- auto LaneClassOf = [&](const TargetRegisterClass *RC) {
- if (TRI->getRegSizeInBits(*RC) <= 32)
- return RC;
- const TargetRegisterClass *Sub =
- TRI->getSubRegisterClass(RC, AMDGPU::sub0);
- return Sub ? Sub : RC;
- };
- const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(VScalarOp);
- const TargetRegisterClass *RFLSrcRC = TRI->getRegClass(TII.getOpRegClassID(
- TII.get(AMDGPU::V_READFIRSTLANE_B32).operands()[1]));
- const TargetRegisterClass *LaneRC = LaneClassOf(VScalarOpRC);
- if (RFLSrcRC && TRI->getCommonSubClass(LaneRC, RFLSrcRC) != LaneRC) {
- const TargetRegisterClass *DemoteRC =
- TRI->getEquivalentVGPRClass(VScalarOpRC);
- assert(DemoteRC &&
- TRI->getCommonSubClass(LaneClassOf(DemoteRC), RFLSrcRC) ==
- LaneClassOf(DemoteRC) &&
- "demote target lane class incompatible with V_READFIRSTLANE_B32");
- Register VRReg = MRI.createVirtualRegister(DemoteRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
- VScalarOp = VRReg;
- }
+ const TargetRegisterClass *RFLSrcRC =
+ TII.getRegClass(TII.get(AMDGPU::V_READFIRSTLANE_B32), 1);
if (NumSubRegs == 1) {
+ const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(VScalarOp);
+ if (const TargetRegisterClass *Common =
+ TRI->getCommonSubClass(VScalarOpRC, RFLSrcRC);
+ Common && Common != VScalarOpRC) {
+ Register VRReg = MRI.createVirtualRegister(Common);
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
+ VScalarOp = VRReg;
+ }
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
>From bbebe17e8f1eeeba402e32f4e6c511d9acc394bf Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 7 May 2026 04:29:19 -0400
Subject: [PATCH 4/6] no-hoist
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 32 ------
.../AMDGPU/waterfall-call-inreg-agpr.mir | 97 +++++++++++++++++++
2 files changed, 97 insertions(+), 32 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d2fb4e3bee71c..6c59238d444e3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7839,38 +7839,6 @@ void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
MI->definesRegister(End->getOperand(1).getReg(), &RI))
++End;
- MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
- auto DominatesStart = [&](const MachineInstr *Q) -> bool {
- if (!Q)
- return false;
- if (MDT)
- return MDT->dominates(Q, &*Start);
- if (Q->getParent() != &MBB)
- return true;
- for (auto It = MBB.begin(); It != Start; ++It)
- if (&*It == Q)
- return true;
- return false;
- };
- for (MachineOperand *MO : ScalarOps) {
- if (!MO->isReg())
- continue;
- Register R = MO->getReg();
- if (!R.isVirtual())
- continue;
- MachineInstr *DefMI = MRI.getVRegDef(R);
- if (!DefMI || !DefMI->isCopy() || DefMI->getParent() != &MBB ||
- DominatesStart(DefMI))
- continue;
- const MachineOperand &Src = DefMI->getOperand(1);
- if (!Src.isReg() || !Src.getReg().isVirtual() ||
- !DominatesStart(MRI.getVRegDef(Src.getReg())))
- continue;
- for (MachineOperand &U : DefMI->all_uses())
- MRI.clearKillFlags(U.getReg());
- MBB.splice(Start, &MBB, DefMI->getIterator());
- }
-
generateWaterFallLoop(*this, *MI, ScalarOps, MDT, Start, End, PhySGPRs);
}
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
new file mode 100644
index 0000000000000..446479e61595a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
@@ -0,0 +1,97 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s
+
+# Regression for the gfx90a waterfall lane reader: when an inreg call
+# argument's source virtual register lives in an AV_* class (which
+# allows AGPR), emitLoadScalarOpsFromVGPRLoop must legalize the
+# ScalarOp into a class compatible with V_READFIRSTLANE_B32 by
+# inserting a COPY to the common subclass (e.g. VGPR_32). Without the
+# COPY, the loop emits `V_READFIRSTLANE_B32 %vreg:av_32`, and
+# `-verify-machineinstrs` fails with:
+# *** Bad machine code: Illegal virtual register for instruction ***
+# Expected a VGPR_32 register, but got a AV_32 register
+
+--- |
+ declare void @callee()
+ define void @waterfall_si_call_inreg_av32() #0 { ret void }
+ define void @waterfall_si_call_inreg_vgpr32() #0 { ret void }
+ attributes #0 = { "target-cpu"="gfx90a" }
+...
+
+# AV_32 ScalarOp: a COPY to VGPR_32 must be inserted in the loop header
+# before V_READFIRSTLANE_B32 reads the lane, so the readfirstlane
+# source is a pure VGPR class.
+
+---
+name: waterfall_si_call_inreg_av32
+tracksRegLiveness: true
+frameInfo:
+ adjustsStack: true
+ hasCalls: true
+machineFunctionInfo:
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: waterfall_si_call_inreg_av32
+ ; CHECK: bb.0:
+ ; CHECK: [[SRC:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK: [[DEST:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+ ; CHECK: bb.1:
+ ; CHECK: [[VCOPY:%[0-9]+]]:vgpr_32 = COPY [[SRC]]
+ ; CHECK: [[LANE:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[VCOPY]], implicit $exec
+ ; CHECK: {{.*}}:sreg_64_xexec = V_CMP_EQ_U32_e64 [[LANE]], [[VCOPY]], implicit $exec
+ ; CHECK: {{.*}}:sreg_64_xexec = S_AND_SAVEEXEC_B64
+ ; CHECK: bb.2:
+ ; CHECK: ADJCALLSTACKUP
+ ; CHECK: $sgpr16 = COPY [[LANE]]
+ ; CHECK: SI_CALL_ISEL [[DEST]], @callee, csr_amdgpu_gfx90ainsts, implicit killed $sgpr16
+ ; CHECK: ADJCALLSTACKDOWN
+ ; CHECK: SI_WATERFALL_LOOP %bb.1
+ bb.0:
+ liveins: $vgpr0, $sgpr4_sgpr5
+
+ %0:av_32 = COPY $vgpr0
+ %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+
+ ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ $sgpr16 = COPY %0
+ SI_CALL_ISEL killed %1, @callee, csr_amdgpu_gfx90ainsts, implicit $sgpr16
+ ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ S_ENDPGM 0
+...
+
+# Baseline: a VGPR_32 ScalarOp is already a legal V_READFIRSTLANE_B32
+# source, so no extra COPY should appear in the loop header and
+# V_READFIRSTLANE_B32 should read the original VGPR directly.
+
+---
+name: waterfall_si_call_inreg_vgpr32
+tracksRegLiveness: true
+frameInfo:
+ adjustsStack: true
+ hasCalls: true
+machineFunctionInfo:
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+ occupancy: 8
+body: |
+ ; CHECK-LABEL: name: waterfall_si_call_inreg_vgpr32
+ ; CHECK: bb.0:
+ ; CHECK: [[SRC:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK: [[DEST:%[0-9]+]]:sreg_64_xexec = COPY $sgpr4_sgpr5
+ ; CHECK: bb.1:
+ ; CHECK-NOT: vgpr_32 = COPY [[SRC]]
+ ; CHECK: [[LANE:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[SRC]], implicit $exec
+ ; CHECK: {{.*}}:sreg_64_xexec = V_CMP_EQ_U32_e64 [[LANE]], [[SRC]], implicit $exec
+ bb.0:
+ liveins: $vgpr0, $sgpr4_sgpr5
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:sreg_64_xexec = COPY $sgpr4_sgpr5
+
+ ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ $sgpr16 = COPY %0
+ SI_CALL_ISEL killed %1, @callee, csr_amdgpu_gfx90ainsts, implicit $sgpr16
+ ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+ S_ENDPGM 0
+...
>From 19b770d23f97a56c1a6d412b25cf0f7ec15c6d55 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 7 May 2026 04:30:51 -0400
Subject: [PATCH 5/6] test-remove
---
.../AMDGPU/waterfall-call-inreg-agpr.ll | 137 ------------------
1 file changed, 137 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
deleted file mode 100644
index 39ac9616ee31b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
+++ /dev/null
@@ -1,137 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s
-
-; gfx90a: AV AGPR sources must be copied to VGPR before V_READFIRSTLANE /
-; V_CMP_EQ in the waterfall; defs moved into BodyBB must be hoisted so LoopBB
-; sees them.
-
- at G = global ptr poison
- at G.1 = global ptr poison
- at G.2 = global ptr addrspace(1) poison
- at G.3 = global ptr addrspace(3) poison
- at G.4 = global ptr addrspace(5) poison
- at G.5 = global ptr addrspace(5) poison
-
-declare amdgpu_cs_chain_preserve void @callee(ptr inreg, ptr,
- ptr addrspace(1) inreg,
- ptr addrspace(1),
- ptr addrspace(3) inreg,
- ptr addrspace(3),
- ptr addrspace(5) inreg,
- ptr addrspace(5))
-
-define amdgpu_cs_chain_preserve void @chain_preserve_caller(float inreg %a, float %b) {
-; CHECK-LABEL: chain_preserve_caller:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_getpc_b64 s[2:3]
-; CHECK-NEXT: s_add_u32 s2, s2, G.5 at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s3, s3, G.5 at gotpcrel32@hi+12
-; CHECK-NEXT: s_mov_b64 s[52:53], s[4:5]
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
-; CHECK-NEXT: s_getpc_b64 s[4:5]
-; CHECK-NEXT: s_add_u32 s4, s4, G.4 at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s5, s5, G.4 at gotpcrel32@hi+12
-; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
-; CHECK-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT: s_getpc_b64 s[6:7]
-; CHECK-NEXT: s_add_u32 s6, s6, G.3 at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s7, s7, G.3 at gotpcrel32@hi+12
-; CHECK-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; CHECK-NEXT: s_getpc_b64 s[2:3]
-; CHECK-NEXT: s_add_u32 s2, s2, G.2 at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s3, s3, G.2 at gotpcrel32@hi+12
-; CHECK-NEXT: flat_load_dword a34, v[0:1]
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
-; CHECK-NEXT: s_getpc_b64 s[4:5]
-; CHECK-NEXT: s_add_u32 s4, s4, G.1 at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s5, s5, G.1 at gotpcrel32@hi+12
-; CHECK-NEXT: flat_load_dword v2, v[0:1]
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
-; CHECK-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT: s_getpc_b64 s[6:7]
-; CHECK-NEXT: s_add_u32 s6, s6, G at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s7, s7, G at gotpcrel32@hi+12
-; CHECK-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x0
-; CHECK-NEXT: flat_load_dword a35, v[0:1]
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; CHECK-NEXT: flat_load_dwordx2 a[32:33], v[0:1]
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
-; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; CHECK-NEXT: v_add_f32_e32 v3, s0, v8
-; CHECK-NEXT: s_mov_b32 s54, s15
-; CHECK-NEXT: s_mov_b32 s55, s14
-; CHECK-NEXT: s_mov_b32 s64, s13
-; CHECK-NEXT: s_mov_b32 s65, s12
-; CHECK-NEXT: s_mov_b64 s[34:35], s[10:11]
-; CHECK-NEXT: s_mov_b64 s[36:37], s[8:9]
-; CHECK-NEXT: s_mov_b64 s[0:1], exec
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dword v[0:1], v3
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_readfirstlane_b32 s17, v2
-; CHECK-NEXT: v_readfirstlane_b32 s3, v5
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s17, v2
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v5
-; CHECK-NEXT: v_readfirstlane_b32 s2, v4
-; CHECK-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s2, v4
-; CHECK-NEXT: s_and_b64 s[4:5], s[0:1], vcc
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s1, v1
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; CHECK-NEXT: s_and_saveexec_b64 s[66:67], s[4:5]
-; CHECK-NEXT: s_getpc_b64 s[4:5]
-; CHECK-NEXT: s_add_u32 s4, s4, callee at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s5, s5, callee at gotpcrel32@hi+12
-; CHECK-NEXT: s_load_dwordx2 s[18:19], s[4:5], 0x0
-; CHECK-NEXT: v_readfirstlane_b32 s16, v0
-; CHECK-NEXT: s_mov_b64 s[4:5], s[52:53]
-; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39]
-; CHECK-NEXT: s_mov_b64 s[8:9], s[36:37]
-; CHECK-NEXT: s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT: s_mov_b32 s12, s65
-; CHECK-NEXT: s_mov_b32 s13, s64
-; CHECK-NEXT: s_mov_b32 s14, s55
-; CHECK-NEXT: s_mov_b32 s15, s54
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a35
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a34
-; CHECK-NEXT: ; implicit-def: $vgpr2
-; CHECK-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; CHECK-NEXT: ; implicit-def: $vgpr2
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr31
-; CHECK-NEXT: ; implicit-def: $vgpr3
-; CHECK-NEXT: s_xor_b64 exec, exec, s[66:67]
-; CHECK-NEXT: s_cbranch_execnz .LBB0_1
-; CHECK-NEXT: ; %bb.2:
-; CHECK-NEXT: s_endpgm
- %LGV6 = load ptr addrspace(5), ptr @G.5, align 8
- %LGV5 = load ptr addrspace(5), ptr @G.4, align 8
- %LGV4 = load ptr addrspace(3), ptr @G.3, align 8
- %LGV2 = load ptr addrspace(1), ptr @G.2, align 8
- %LGV1 = load ptr, ptr @G.1, align 8
- %LGV = load ptr, ptr @G, align 8
- %c = fadd float %a, %b
- store float %c, ptr poison, align 4
- call void @callee(ptr %LGV, ptr %LGV1,
- ptr addrspace(1) %LGV2, ptr addrspace(1) poison,
- ptr addrspace(3) poison, ptr addrspace(3) %LGV4,
- ptr addrspace(5) %LGV5, ptr addrspace(5) %LGV6)
- ret void
-}
>From 1418c56b725259d6692b618665fc37a59b53ffd9 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 14 May 2026 03:02:21 -0400
Subject: [PATCH 6/6] ir-test
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
.../AMDGPU/waterfall-call-inreg-agpr.ll | 65 +++++++++++++++++++
.../AMDGPU/waterfall-call-inreg-agpr.mir | 5 +-
3 files changed, 68 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6c59238d444e3..afccf9412303d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7116,7 +7116,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(VScalarOp);
if (const TargetRegisterClass *Common =
TRI->getCommonSubClass(VScalarOpRC, RFLSrcRC);
- Common && Common != VScalarOpRC) {
+ Common != VScalarOpRC) {
Register VRReg = MRI.createVirtualRegister(Common);
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
VScalarOp = VRReg;
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
new file mode 100644
index 0000000000000..fc61f15c42f03
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck %s
+
+; On gfx90a, a 32-bit FLAT load may land in AV_32 (VGPR/AGPR). When passed
+; inreg to an amdgpu_cs_chain_preserve call, SIInstrInfo must copy the
+; ScalarOp into a VGPR-only class before v_readfirstlane_b32; otherwise
+; -verify-machineinstrs rejects it.
+
+ at G = global ptr addrspace(3) poison
+
+declare amdgpu_cs_chain_preserve void @callee(ptr addrspace(3) inreg, ptr addrspace(3))
+
+define amdgpu_cs_chain_preserve void @caller() {
+; CHECK-LABEL: caller:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_getpc_b64 s[0:1]
+; CHECK-NEXT: s_add_u32 s0, s0, G at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s1, s1, G at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; CHECK-NEXT: s_mov_b32 s54, s15
+; CHECK-NEXT: s_mov_b32 s55, s14
+; CHECK-NEXT: s_mov_b32 s64, s13
+; CHECK-NEXT: s_mov_b32 s65, s12
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; CHECK-NEXT: flat_load_dword v0, v[0:1]
+; CHECK-NEXT: s_mov_b64 s[34:35], s[10:11]
+; CHECK-NEXT: s_mov_b64 s[36:37], s[8:9]
+; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
+; CHECK-NEXT: s_mov_b64 s[52:53], s[4:5]
+; CHECK-NEXT: s_mov_b64 s[0:1], exec
+; CHECK-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_readfirstlane_b32 s16, v0
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s16, v0
+; CHECK-NEXT: s_and_saveexec_b64 s[66:67], vcc
+; CHECK-NEXT: s_getpc_b64 s[0:1]
+; CHECK-NEXT: s_add_u32 s0, s0, callee at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s1, s1, callee at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[18:19], s[0:1], 0x0
+; CHECK-NEXT: s_mov_b64 s[0:1], s[48:49]
+; CHECK-NEXT: s_mov_b64 s[2:3], s[50:51]
+; CHECK-NEXT: s_mov_b64 s[4:5], s[52:53]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39]
+; CHECK-NEXT: s_mov_b64 s[8:9], s[36:37]
+; CHECK-NEXT: s_mov_b64 s[10:11], s[34:35]
+; CHECK-NEXT: s_mov_b32 s12, s65
+; CHECK-NEXT: s_mov_b32 s13, s64
+; CHECK-NEXT: s_mov_b32 s14, s55
+; CHECK-NEXT: s_mov_b32 s15, s54
+; CHECK-NEXT: s_mov_b32 s0, s16
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr31
+; CHECK-NEXT: s_xor_b64 exec, exec, s[66:67]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_1
+; CHECK-NEXT: ; %bb.2:
+; CHECK-NEXT: s_endpgm
+ %p = load ptr addrspace(3), ptr @G, align 4
+ call void @callee(ptr addrspace(3) %p, ptr addrspace(3) poison)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
index 446479e61595a..f23c332b85fb1 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-inreg-agpr.mir
@@ -12,9 +12,8 @@
--- |
declare void @callee()
- define void @waterfall_si_call_inreg_av32() #0 { ret void }
- define void @waterfall_si_call_inreg_vgpr32() #0 { ret void }
- attributes #0 = { "target-cpu"="gfx90a" }
+ define void @waterfall_si_call_inreg_av32() { ret void }
+ define void @waterfall_si_call_inreg_vgpr32() { ret void }
...
# AV_32 ScalarOp: a COPY to VGPR_32 must be inserted in the loop header
More information about the llvm-commits
mailing list