[llvm] [AMDGPU] Recognize more trivially disjoint LDS memory accesses (PR #191599)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 03:23:02 PDT 2026
https://github.com/xindubawukong updated https://github.com/llvm/llvm-project/pull/191599
>From 4f9aa16176f36f0eef5aed073f942b17f91e4c40 Mon Sep 17 00:00:00 2001
From: xdbwk <xindubawukong at gmail.com>
Date: Sat, 11 Apr 2026 04:20:02 +0000
Subject: [PATCH 1/2] [AMDGPU] Recognize more trivially disjoint LDS memory
accesses
The AMDGPU scheduler currently misses some cases where memory accesses
can be proven trivially disjoint, which prevents legal and profitable
reordering.
There are two main issues:
1) checkInstOffsetsDoNotOverlap only handles accesses whose base
MachineOperands match directly. This misses cases where the two
instructions access the same underlying LDS object but use different
IR-derived base values. In such cases, the corresponding
MachineMemOperands may still carry enough information to recover a
common base with constant offsets.
2) DS and LDSDMA instructions are treated conservatively whenever either
instruction is an LDSDMA op (see #75249), even when their LDS-side
MachineMemOperands clearly refer to non-overlapping ranges of the same
object.
This change addresses both issues:
- When base operands do not match, fall back to recovering the underlying
base pointer and constant offsets from the MachineMemOperands using
GetPointerBaseWithConstantOffset(). If both accesses refer to the same
underlying object and their ranges do not overlap, they are considered
trivially disjoint.
- For DS/LDSDMA pairs, allow disambiguation when both instructions expose
a unique LDS MachineMemOperand and the accessed ranges are provably
non-overlapping.
The fallback is only applied when sufficient information is available
from the MachineMemOperands; otherwise, the existing conservative
behavior is preserved.
Added tests cover:
- DS vs LDSDMA disambiguation using LDS-side MachineMemOperands
- accesses with different MachineOperand bases but the same underlying
LDS object and non-overlapping constant offsets
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 111 ++++++++++++++++--
.../triv-disjoint-lds-dma-local-access.mir | 40 +++++++
...-disjoint-mem-access-underlying-offset.mir | 43 +++++++
3 files changed, 183 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/triv-disjoint-lds-dma-local-access.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-underlying-offset.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3179c65340d18..79c842bf0ea88 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -533,6 +533,51 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return false;
}
+static bool getBasePtrAndOffset(const MachineInstr &MI,
+ const MachineMemOperand &MMO,
+ const Value *&Base,
+ int64_t &Offset) {
+ Offset = MMO.getOffset();
+ Base = nullptr;
+
+ const Value *V = MMO.getValue();
+ if (!V)
+ return false;
+
+ Base = GetPointerBaseWithConstantOffset(V, Offset, MI.getMF()->getDataLayout(),
+ /*AllowNonInbounds=*/true);
+ if (!Base)
+ return false;
+
+ Base = getUnderlyingObject(Base);
+ return !isa<UndefValue>(Base);
+}
+
+static bool memOperandsHaveSameBasePtr(const MachineInstr &MI1,
+ const MachineMemOperand &MMO1,
+ const MachineInstr &MI2,
+ const MachineMemOperand &MMO2,
+ int64_t *Offset1,
+ int64_t *Offset2) {
+ if (MMO1.getAddrSpace() != MMO2.getAddrSpace())
+ return false;
+
+ const Value *Base1 = nullptr;
+ const Value *Base2 = nullptr;
+ int64_t BaseOffset1 = 0;
+ int64_t BaseOffset2 = 0;
+ if (!getBasePtrAndOffset(MI1, MMO1, Base1, BaseOffset1) ||
+ !getBasePtrAndOffset(MI2, MMO2, Base2, BaseOffset2))
+ return false;
+
+ if (Base1 != Base2)
+ return false;
+
+ *Offset1 = BaseOffset1;
+ *Offset2 = BaseOffset2;
+ return true;
+}
+
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
ArrayRef<const MachineOperand *> BaseOps1,
const MachineInstr &MI2,
@@ -4033,13 +4078,43 @@ memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
return true;
}
-static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
- LocationSize WidthB, int OffsetB) {
- int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
- int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
+static bool offsetsDoNotOverlap(LocationSize WidthA, int64_t OffsetA,
+ LocationSize WidthB, int64_t OffsetB) {
+ int64_t LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
+ int64_t HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
return LowWidth.hasValue() &&
- LowOffset + (int)LowWidth.getValue() <= HighOffset;
+ LowOffset + static_cast<int64_t>(LowWidth.getValue()) <= HighOffset;
+}
+
+static const MachineMemOperand *getLDSMemOperand(const MachineInstr &MI) {
+ // Return the single LDS MachineMemOperand. Zero or multiple LDS operands are
+ // ambiguous for this disambiguation, so handle them conservatively.
+ const MachineMemOperand *LDSMMO = nullptr;
+ for (const MachineMemOperand *MMO : MI.memoperands()) {
+ if (MMO->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
+ continue;
+ if (LDSMMO)
+ return nullptr;
+ LDSMMO = MMO;
+ }
+ return LDSMMO;
+}
+
+static bool checkLDSMemOpsDoNotOverlap(const MachineInstr &MIa,
+ const MachineInstr &MIb) {
+ const MachineMemOperand *MMO0 = getLDSMemOperand(MIa);
+ const MachineMemOperand *MMO1 = getLDSMemOperand(MIb);
+ if (!MMO0 || !MMO1)
+ return false;
+
+ int64_t Offset0 = 0;
+ int64_t Offset1 = 0;
+ if (!memOperandsHaveSameBasePtr(MIa, *MMO0, MIb, *MMO1, &Offset0, &Offset1))
+ return false;
+
+ return offsetsDoNotOverlap(MMO0->getSize(), Offset0, MMO1->getSize(),
+ Offset1);
}
bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
@@ -4055,15 +4130,26 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
Dummy1, &RI))
return false;
- if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
- return false;
-
if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
// FIXME: Handle ds_read2 / ds_write2.
return false;
}
- LocationSize Width0 = MIa.memoperands().front()->getSize();
- LocationSize Width1 = MIb.memoperands().front()->getSize();
+
+ auto *MMO0 = MIa.memoperands().front();
+ auto *MMO1 = MIb.memoperands().front();
+ LocationSize Width0 = MMO0->getSize();
+ LocationSize Width1 = MMO1->getSize();
+
+ if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) {
+ int64_t BaseOffset0 = 0;
+ int64_t BaseOffset1 = 0;
+ if (!memOperandsHaveSameBasePtr(MIa, *MMO0, MIb, *MMO1, &BaseOffset0,
+ &BaseOffset1))
+ return false;
+ Offset0 += BaseOffset0;
+ Offset1 += BaseOffset1;
+ }
+
return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
}
@@ -4081,8 +4167,11 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
return false;
- if (isLDSDMA(MIa) || isLDSDMA(MIb))
+ if (isLDSDMA(MIa) || isLDSDMA(MIb)) {
+ if ((isDS(MIa) && isLDSDMA(MIb)) || (isLDSDMA(MIa) && isDS(MIb)))
+ return checkLDSMemOpsDoNotOverlap(MIa, MIb);
return false;
+ }
if (MIa.isBundle() || MIb.isBundle())
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/triv-disjoint-lds-dma-local-access.mir b/llvm/test/CodeGen/AMDGPU/triv-disjoint-lds-dma-local-access.mir
new file mode 100644
index 0000000000000..65160073c14e5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/triv-disjoint-lds-dma-local-access.mir
@@ -0,0 +1,40 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx940 -verify-machineinstrs -enable-misched -run-pass=machine-scheduler -o - %s 2>&1 | FileCheck %s
+
+# Make sure DS accesses can be proven disjoint from LDSDMA accesses using the
+# LDS-side MachineMemOperands, so the scheduler can move the DS access ahead of
+# the LDSDMA operation.
+
+--- |
+ @lds = external addrspace(3) global [4 x i32], align 4
+
+ define amdgpu_kernel void @lds_dma_local_access_disjoint(i32 addrspace(1)* %src) {
+ %lds.ptr = getelementptr [4 x i32], [4 x i32] addrspace(3)* @lds, i32 0, i32 0
+ %lds.ptr.2 = getelementptr [4 x i32], [4 x i32] addrspace(3)* @lds, i32 0, i32 2
+ %val = load i32, i32 addrspace(1)* %src
+ store i32 %val, i32 addrspace(3)* %lds.ptr, align 4
+ %lds.ptr.2.i64 = bitcast i32 addrspace(3)* %lds.ptr.2 to i64 addrspace(3)*
+ %res = load i64, i64 addrspace(3)* %lds.ptr.2.i64, align 4
+ ret void
+ }
+
+# CHECK-LABEL: name: lds_dma_local_access_disjoint
+# CHECK: %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+# CHECK-NEXT: dead %3:vreg_64_align2 = DS_READ_B64_gfx9 %2, 8, 0, implicit $exec
+# CHECK-NEXT: $m0 = S_MOV_B32 0
+# CHECK-NEXT: BUFFER_LOAD_DWORD_LDS_OFFEN %2, %0, %1, 0, 0, 0, implicit $exec, implicit $m0
+---
+name: lds_dma_local_access_disjoint
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4
+
+ $m0 = S_MOV_B32 0
+ %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+ %1:sreg_32 = COPY $sgpr4
+ %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ BUFFER_LOAD_DWORD_LDS_OFFEN %2, %0, %1, 0, 0, 0, implicit $exec, implicit $m0 :: (load (s32) from %ir.src, align 4, addrspace 1), (store (s32) into %ir.lds.ptr, align 4, addrspace 3)
+ %3:vreg_64_align2 = DS_READ_B64_gfx9 %2, 8, 0, implicit $exec :: (load (s64) from %ir.lds.ptr.2.i64, align 4, addrspace 3)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-underlying-offset.mir b/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-underlying-offset.mir
new file mode 100644
index 0000000000000..3e91bce8d1e6a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-underlying-offset.mir
@@ -0,0 +1,43 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -enable-misched -run-pass=machine-scheduler -o - -debug-only=machine-scheduler %s 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Make sure DS accesses with different MachineOperand bases but the same
+# underlying LDS object are treated as trivially disjoint after accounting for
+# the constant offsets recovered from the MachineMemOperands.
+
+# CHECK: SU(4): %3:vreg_128 = DS_READ_B128_gfx9 %1:vgpr_32, 0, 0, implicit $exec :: (load (s128) from %ir.load.ptr, addrspace 3)
+# CHECK: Predecessors:
+# CHECK-NEXT: SU(1): Out Latency=1
+# CHECK-NEXT: SU(1): Data Latency=1 Reg=%1
+# CHECK-NOT: SU(3): Ord
+
+--- |
+ @lds = internal addrspace(3) global [8192 x i32] undef, align 16
+ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+
+ define amdgpu_kernel void @test() {
+ entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %tid.copy = add i32 %tid, 0
+ %store.base = getelementptr [8192 x i32], ptr addrspace(3) @lds, i32 0, i32 %tid
+ %load.base = getelementptr [8192 x i32], ptr addrspace(3) @lds, i32 0, i32 %tid.copy
+ %store.ptr = getelementptr i32, ptr addrspace(3) %store.base, i32 4096
+ %load.ptr = getelementptr i32, ptr addrspace(3) %load.base, i32 4352
+ ret void
+ }
+
+...
+---
+name: test
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ liveins: $vgpr0
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %2:vreg_128 = IMPLICIT_DEF
+ DS_WRITE_B128_gfx9 %0, %2, 0, 0, implicit $exec :: (store (s128) into %ir.store.ptr, addrspace 3)
+ %3:vreg_128 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128) from %ir.load.ptr, addrspace 3)
+ S_ENDPGM 0
+...
>From b809593309a8f58df6e4009aef48f4b54042bbf5 Mon Sep 17 00:00:00 2001
From: xdbwk <xindubawukong at gmail.com>
Date: Tue, 14 Apr 2026 10:19:49 +0000
Subject: [PATCH 2/2] fix clang-format
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 79c842bf0ea88..4815a0284284d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -535,8 +535,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
static bool getBasePtrAndOffset(const MachineInstr &MI,
const MachineMemOperand &MMO,
- const Value *&Base,
- int64_t &Offset) {
+ const Value *&Base, int64_t &Offset) {
Offset = MMO.getOffset();
Base = nullptr;
@@ -544,8 +543,9 @@ static bool getBasePtrAndOffset(const MachineInstr &MI,
if (!V)
return false;
- Base = GetPointerBaseWithConstantOffset(V, Offset, MI.getMF()->getDataLayout(),
- /*AllowNonInbounds=*/true);
+ Base =
+ GetPointerBaseWithConstantOffset(V, Offset, MI.getMF()->getDataLayout(),
+ /*AllowNonInbounds=*/true);
if (!Base)
return false;
@@ -557,8 +557,7 @@ static bool memOperandsHaveSameBasePtr(const MachineInstr &MI1,
const MachineMemOperand &MMO1,
const MachineInstr &MI2,
const MachineMemOperand &MMO2,
- int64_t *Offset1,
- int64_t *Offset2) {
+ int64_t *Offset1, int64_t *Offset2) {
if (MMO1.getAddrSpace() != MMO2.getAddrSpace())
return false;
More information about the llvm-commits
mailing list