[llvm] [AMDGPU] Do not forward reserved aperture registers into wider copies (PR #220517)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 12:41:38 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/220517
>From 26cdf4980548bc8ad3f5a3c270730f6bfbc0392f Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 2 Sep 2026 10:41:17 +0200
Subject: [PATCH 1/3] [AMDGPU] Do not forward reserved aperture registers into
wider copies
src_shared_base/src_private_base have no addressable high half, so MachineCopyPropagation must not forward them into a copy that later splits into subregisters
SIInstrInfo::copyPhysReg now reports an error instead of asserting if asked to do so anyway
---
llvm/lib/CodeGen/MachineCopyPropagation.cpp | 22 +++++++++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 9 ++++++
.../AMDGPU/illegal-aperture-reg-copy.mir | 14 ++++++++
.../CodeGen/AMDGPU/machine-cp-aperture-reg.ll | 15 +++++++++
.../CodeGen/AMDGPU/machine-cp-propagation.mir | 32 +++++++++++++++++++
5 files changed, 92 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 0568890583c1d..2d165e7c59a47 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -821,6 +821,20 @@ bool MachineCopyPropagation::canUpdateSrcUsers(const MachineInstr &Copy,
return true;
}
+/// Return true if \p NewReg has a non-artificial subregister for every
+/// subregister index where \p OldReg has one.
+static bool hasMatchingSubRegs(MCRegister OldReg, MCRegister NewReg,
+ const TargetRegisterInfo *TRI) {
+ for (MCSubRegIndexIterator SRI(OldReg, TRI); SRI.isValid(); ++SRI) {
+ if (TRI->isArtificial(SRI.getSubReg()))
+ continue;
+ MCRegister Sub = TRI->getSubReg(NewReg, SRI.getSubRegIndex());
+ if (!Sub || TRI->isArtificial(Sub))
+ return false;
+ }
+ return true;
+}
+
/// Look for available copies whose destination register is used by \p MI and
/// replace the use in \p MI with the copy's source register.
void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
@@ -879,6 +893,14 @@ void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
if (MRI->isReserved(CopySrc) && !MRI->isConstantPhysReg(CopySrc))
continue;
+ // A reserved source may lack subregs the copy is later split into.
+ if (MRI->isReserved(CopySrc) && isCopyInstr(MI, *TII, UseCopyInstr) &&
+ !hasMatchingSubRegs(MOUse.getReg().asMCReg(), ForwardedReg, TRI)) {
+ LLVM_DEBUG(dbgs() << "MCP: Copy source is missing subregisters of "
+ << printReg(MOUse.getReg(), TRI) << '\n');
+ continue;
+ }
+
if (!isForwardableRegClassCopy(*Copy, MI, OpIdx))
continue;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3094c1390e9cf..db3fdf0ea8147 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1189,6 +1189,15 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
+ // Report rather than assert if a register is missing a required subreg.
+ for (int16_t SubIdx : SubIndices) {
+ if (!RI.getSubReg(DestReg, SubIdx) || !RI.getSubReg(SrcReg, SubIdx)) {
+ reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
+ "Cannot decompose copy into subregister moves!");
+ return;
+ }
+ }
+
// If there is an overlap, we can't kill the super-register on the last
// instruction, since it will also kill the components made live by this def.
const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir b/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
new file mode 100644
index 0000000000000..e3c5ac9abbe7f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
@@ -0,0 +1,14 @@
+# RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -run-pass=postrapseudos -o /dev/null %s 2>&1 | FileCheck %s
+
+# src_shared_base has no addressable high half. copyPhysReg must not assert.
+
+# CHECK: error: {{.*}} in function illegal_aperture_reg_copy void (): Cannot decompose copy into subregister moves!
+
+---
+name: illegal_aperture_reg_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ renamable $vgpr0_vgpr1 = COPY $src_shared_base
+ S_ENDPGM 0, implicit $vgpr0_vgpr1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
new file mode 100644
index 0000000000000..9e44e9e72973b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
@@ -0,0 +1,15 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel < %s | FileCheck %s
+
+; src_shared_base has no addressable high half. Must not forward into VGPRs.
+
+ at lds = internal addrspace(3) global [32 x float] poison, align 4
+
+; CHECK-LABEL: aperture_copy_to_vgpr_pair:
+; CHECK: s_mov_b64 s[{{[0-9]+}}:{{[0-9]+}}], src_shared_base
+define float @aperture_copy_to_vgpr_pair(i32 %i) {
+ %idx = zext i32 %i to i64
+ %gep = getelementptr float, ptr addrspacecast (ptr addrspace(3) @lds to ptr), i64 %idx
+ %cast = addrspacecast ptr %gep to ptr addrspace(3)
+ %v = load float, ptr addrspace(3) %cast, align 4
+ ret float %v
+}
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir b/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
index f8630245b6938..8cbd8cac4c588 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
@@ -34,3 +34,35 @@ body: |
S_ENDPGM 0
...
+
+# src_shared_base has no addressable high half. machine-cp must not forward it.
+
+---
+name: no_forward_aperture_into_vgpr_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: no_forward_aperture_into_vgpr_copy
+ ; CHECK: renamable $sgpr0_sgpr1 = COPY $src_shared_base
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0_vgpr1
+ renamable $sgpr0_sgpr1 = COPY $src_shared_base
+ renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+ S_ENDPGM 0, implicit $vgpr0_vgpr1
+...
+
+---
+name: forward_sgpr_into_vgpr_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ ; CHECK-LABEL: name: forward_sgpr_into_vgpr_copy
+ ; CHECK: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0_vgpr1
+ renamable $sgpr0_sgpr1 = COPY $sgpr4_sgpr5
+ renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+ S_ENDPGM 0, implicit $vgpr0_vgpr1
+...
>From a646fee9f139cba7073e4913a14b50509d179782 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 8 Sep 2026 20:18:39 +0200
Subject: [PATCH 2/3] comments
---
llvm/lib/CodeGen/MachineCopyPropagation.cpp | 33 ++++++++++---------
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 31 ++++++++++++-----
.../AMDGPU/illegal-aperture-reg-copy.mir | 14 --------
.../CodeGen/AMDGPU/machine-cp-aperture-reg.ll | 5 +--
.../CodeGen/AMDGPU/machine-cp-propagation.mir | 14 ++++----
.../AMDGPU/verify-copy-subreg-split.mir | 32 ++++++++++++++++++
6 files changed, 80 insertions(+), 49 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
create mode 100644 llvm/test/MachineVerifier/AMDGPU/verify-copy-subreg-split.mir
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 2d165e7c59a47..c824d83828344 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -821,15 +821,14 @@ bool MachineCopyPropagation::canUpdateSrcUsers(const MachineInstr &Copy,
return true;
}
-/// Return true if \p NewReg has a non-artificial subregister for every
-/// subregister index where \p OldReg has one.
+/// Ignores artificial subregisters: copies never split into them, and
+/// requiring a match would reject pairs differing only in artificial shape.
static bool hasMatchingSubRegs(MCRegister OldReg, MCRegister NewReg,
- const TargetRegisterInfo *TRI) {
- for (MCSubRegIndexIterator SRI(OldReg, TRI); SRI.isValid(); ++SRI) {
- if (TRI->isArtificial(SRI.getSubReg()))
+ const TargetRegisterInfo &TRI) {
+ for (MCSubRegIndexIterator SRI(OldReg, &TRI); SRI.isValid(); ++SRI) {
+ if (TRI.isArtificial(SRI.getSubReg()))
continue;
- MCRegister Sub = TRI->getSubReg(NewReg, SRI.getSubRegIndex());
- if (!Sub || TRI->isArtificial(Sub))
+ if (!TRI.getSubReg(NewReg, SRI.getSubRegIndex()))
return false;
}
return true;
@@ -889,16 +888,18 @@ void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
}
}
- // Don't forward COPYs of reserved regs unless they are constant.
- if (MRI->isReserved(CopySrc) && !MRI->isConstantPhysReg(CopySrc))
- continue;
+ if (MRI->isReserved(CopySrc)) {
+ // Don't forward COPYs of reserved regs unless they are constant.
+ if (!MRI->isConstantPhysReg(CopySrc))
+ continue;
- // A reserved source may lack subregs the copy is later split into.
- if (MRI->isReserved(CopySrc) && isCopyInstr(MI, *TII, UseCopyInstr) &&
- !hasMatchingSubRegs(MOUse.getReg().asMCReg(), ForwardedReg, TRI)) {
- LLVM_DEBUG(dbgs() << "MCP: Copy source is missing subregisters of "
- << printReg(MOUse.getReg(), TRI) << '\n');
- continue;
+ // A reserved source may lack subregs the copy is later split into.
+ if (isCopyInstr(MI, *TII, UseCopyInstr) &&
+ !hasMatchingSubRegs(MOUse.getReg().asMCReg(), ForwardedReg, *TRI)) {
+ LLVM_DEBUG(dbgs() << "MCP: Copy source is missing subregisters of "
+ << printReg(MOUse.getReg(), TRI) << '\n');
+ continue;
+ }
}
if (!isForwardableRegClassCopy(*Copy, MI, OpIdx))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2e52c35c4b760..77b49e3b430ad 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1193,15 +1193,6 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, 4);
- // Report rather than assert if a register is missing a required subreg.
- for (int16_t SubIdx : SubIndices) {
- if (!RI.getSubReg(DestReg, SubIdx) || !RI.getSubReg(SrcReg, SubIdx)) {
- reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
- "Cannot decompose copy into subregister moves!");
- return;
- }
- }
-
for (unsigned Idx{}; Idx < SubIndices.size();) {
unsigned NumRegs = 1;
unsigned ThisOpcode = Opcode;
@@ -5316,6 +5307,28 @@ bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
ErrInfo = "illegal copy from vector register to SGPR";
return false;
}
+
+ if (!DstReg.isPhysical() || !SrcReg.isPhysical())
+ return true;
+
+ const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DstReg);
+ const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);
+ if (!RC || !SrcRC || RI.isSGPRClass(RC) || RI.getRegSizeInBits(*RC) <= 32)
+ return true;
+
+ // Copies handled by a single 64-bit move are not decomposed.
+ if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC)) &&
+ (ST.hasVMovB64Inst() || ST.hasPkMovB32()))
+ return true;
+
+ // Other wide vector copies split into per-subregister moves.
+ for (int16_t SubIdx : RI.getRegSplitParts(RC, 4)) {
+ if (!RI.getSubReg(DstReg, SubIdx) || !RI.getSubReg(SrcReg, SubIdx)) {
+ ErrInfo = "cannot decompose copy into subregister moves";
+ return false;
+ }
+ }
+
return true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir b/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
deleted file mode 100644
index e3c5ac9abbe7f..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/illegal-aperture-reg-copy.mir
+++ /dev/null
@@ -1,14 +0,0 @@
-# RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -run-pass=postrapseudos -o /dev/null %s 2>&1 | FileCheck %s
-
-# src_shared_base has no addressable high half. copyPhysReg must not assert.
-
-# CHECK: error: {{.*}} in function illegal_aperture_reg_copy void (): Cannot decompose copy into subregister moves!
-
----
-name: illegal_aperture_reg_copy
-tracksRegLiveness: true
-body: |
- bb.0:
- renamable $vgpr0_vgpr1 = COPY $src_shared_base
- S_ENDPGM 0, implicit $vgpr0_vgpr1
-...
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
index 9e44e9e72973b..0dcddca996662 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
@@ -1,10 +1,11 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck %s
; src_shared_base has no addressable high half. Must not forward into VGPRs.
@lds = internal addrspace(3) global [32 x float] poison, align 4
-; CHECK-LABEL: aperture_copy_to_vgpr_pair:
+; CHECK-LABEL: {{^}}aperture_copy_to_vgpr_pair:
+; CHECK-NOT: v_mov_b32{{.*}}src_shared_base
; CHECK: s_mov_b64 s[{{[0-9]+}}:{{[0-9]+}}], src_shared_base
define float @aperture_copy_to_vgpr_pair(i32 %i) {
%idx = zext i32 %i to i64
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir b/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
index 8cbd8cac4c588..4aaf5bd8a5d8e 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-cp-propagation.mir
@@ -34,7 +34,6 @@ body: |
S_ENDPGM 0
...
-
# src_shared_base has no addressable high half. machine-cp must not forward it.
---
@@ -51,18 +50,17 @@ body: |
S_ENDPGM 0, implicit $vgpr0_vgpr1
...
+# sgpr_null64 is also reserved, but it has both halves, so forwarding is fine.
+
---
-name: forward_sgpr_into_vgpr_copy
+name: forward_null_into_vgpr_copy
tracksRegLiveness: true
body: |
bb.0:
- liveins: $sgpr4_sgpr5
- ; CHECK-LABEL: name: forward_sgpr_into_vgpr_copy
- ; CHECK: liveins: $sgpr4_sgpr5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY $sgpr4_sgpr5
+ ; CHECK-LABEL: name: forward_null_into_vgpr_copy
+ ; CHECK: renamable $vgpr0_vgpr1 = COPY $sgpr_null64
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0_vgpr1
- renamable $sgpr0_sgpr1 = COPY $sgpr4_sgpr5
+ renamable $sgpr0_sgpr1 = COPY $sgpr_null64
renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
S_ENDPGM 0, implicit $vgpr0_vgpr1
...
diff --git a/llvm/test/MachineVerifier/AMDGPU/verify-copy-subreg-split.mir b/llvm/test/MachineVerifier/AMDGPU/verify-copy-subreg-split.mir
new file mode 100644
index 0000000000000..3a0a68f92cef8
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/verify-copy-subreg-split.mir
@@ -0,0 +1,32 @@
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -run-pass=none -filetype=null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code" %s
+# gfx942 expands this into a single V_MOV_B64, so it needs no subregisters.
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=none -filetype=null %s 2>&1 | FileCheck -check-prefix=GFX942 -implicit-check-not="Bad machine code" %s
+
+# src_shared_base has no addressable high half, so the copy cannot be split.
+
+# CHECK: *** Bad machine code: cannot decompose copy into subregister moves ***
+# GFX942: name: illegal_aperture_reg_copy
+
+---
+name: illegal_aperture_reg_copy
+# isSSA: false is required, the COPY check only runs after SSA.
+isSSA: false
+tracksRegLiveness: true
+body: |
+ bb.0:
+ renamable $vgpr0_vgpr1 = COPY $src_shared_base
+ S_ENDPGM 0, implicit $vgpr0_vgpr1
+...
+
+# $pc_reg belongs to no register class, so it has no base class to query.
+
+---
+name: copy_reg_without_base_class
+isSSA: false
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1
+ $pc_reg = COPY $sgpr0_sgpr1
+ S_ENDPGM 0
+...
>From 3b42e5633d68d7bd0a10f4114b3d6e5969f4ad66 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 8 Sep 2026 22:10:15 +0200
Subject: [PATCH 3/3] triples
---
llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
index 0dcddca996662..bcea04d268633 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-cp-aperture-reg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck %s
; src_shared_base has no addressable high half. Must not forward into VGPRs.
More information about the llvm-commits
mailing list