[llvm] [AMDGPU][SILoadStoreOptimizer] Fix lds address operand offset (PR #176816)
Ryan Mitchell via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 22 07:59:03 PST 2026
https://github.com/RyanRio updated https://github.com/llvm/llvm-project/pull/176816
>From 8867e4a1bc70f85e462eff03c33b7bde596ccedd Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Mon, 19 Jan 2026 12:33:52 -0800
Subject: [PATCH 1/4] [AMDGPU][SILoadStoreOptimizer] Fix lds address operand
offset
---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 8 ++
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 30 +++++++-
.../promote-constOffset-to-imm-gfx12.mir | 74 +++++++++++++++++++
3 files changed, 110 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index bf51b22274f3d..44037d01dfdd1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1019,6 +1019,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;
}
+ static bool usesASYNC_CNT(const MachineInstr &MI) {
+ return MI.getDesc().TSFlags & SIInstrFlags::ASYNC_CNT;
+ }
+
+ bool usesASYNC_CNT(uint16_t Opcode) const {
+ return get(Opcode).TSFlags & SIInstrFlags::ASYNC_CNT;
+ }
+
// Most sopk treat the immediate as a signed 16-bit, however some
// use it as unsigned.
static bool sopkIsZext(unsigned Opcode) {
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index dc1592e5fb82a..f50333b5886a7 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -279,6 +279,7 @@ class SILoadStoreOptimizer {
void updateBaseAndOffset(MachineInstr &I, Register NewBase,
int32_t NewOffset) const;
+ void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
std::optional<int32_t> extractConstOffset(const MachineOperand &Op) const;
@@ -2311,6 +2312,27 @@ void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base
Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32);
}
+// Maintain the correct LDS address for async loads.
+// It becomes incorrect when promoteConstantOffsetToImm
+// adds an offset only meant for the src operand.
+void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
+ int32_t OffsetDiff) const {
+ if (!TII->usesASYNC_CNT(MI) || OffsetDiff == 0)
+ return;
+
+ Register OldVDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
+ Register NewVDst =
+ MRI->createVirtualRegister(TRI->getRegClassForReg(*MRI, OldVDst));
+ MachineBasicBlock &MBB = *MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), NewVDst)
+ .addReg(OldVDst)
+ .addImm(-OffsetDiff)
+ .addImm(0);
+
+ MI.getOperand(0).setReg(NewVDst);
+}
+
bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
MachineInstr &MI,
MemInfoMap &Visited,
@@ -2440,7 +2462,9 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
// Instead of moving up, just re-compute anchor-instruction's base address.
Register Base = computeBase(MI, AnchorAddr);
- updateBaseAndOffset(MI, Base, MAddr.Offset - AnchorAddr.Offset);
+ int32_t OffsetDiff = MAddr.Offset - AnchorAddr.Offset;
+ updateBaseAndOffset(MI, Base, OffsetDiff);
+ updateAsyncLDSAddress(MI, OffsetDiff);
LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump(););
for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
@@ -2451,7 +2475,9 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
if (TLI->isLegalFlatAddressingMode(AM, AS)) {
LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
OtherMI->dump());
- updateBaseAndOffset(*OtherMI, Base, OtherOffset - AnchorAddr.Offset);
+ int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
+ updateBaseAndOffset(*OtherMI, Base, OtherOffsetDiff);
+ updateAsyncLDSAddress(*OtherMI, OtherOffsetDiff);
LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
new file mode 100644
index 0000000000000..3f243f39fa585
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs -start-before=si-load-store-opt -stop-before=si-insert-waitcnts --verify-machineinstrs -o - %s | FileCheck -check-prefixes=GFX1250 %s
+
+---
+name: promote_async_load_offset
+machineFunctionInfo:
+ stackPtrOffsetReg: '$sgpr32'
+ frameOffsetReg: '$sgpr33'
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ liveins: $vgpr0, $sgpr0_sgpr1, $ttmp7
+ ; GFX1250-LABEL: name: promote_async_load_offset
+ ; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 512, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, dead $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 256, $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %0:vgpr_32(s32) = COPY $vgpr0
+ %5:vgpr_32 = V_LSHLREV_B32_e64 8, %0:vgpr_32(s32), implicit $exec
+ %7:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
+ %8:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
+ %32:vgpr_32, %34:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 256, 0, implicit $exec
+ %33:vgpr_32, %35:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %34:sreg_32_xm0_xexec, 0, implicit $exec
+ %17:vreg_64_align2 = REG_SEQUENCE %32:vgpr_32, %subreg.sub0, %33:vgpr_32, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %7:vgpr_32, killed %17:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %40:vgpr_32, %36:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 512, 0, implicit $exec
+ %41:vgpr_32, %37:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %36:sreg_32_xm0_xexec, 0, implicit $exec
+ %21:vreg_64_align2 = REG_SEQUENCE %40:vgpr_32, %subreg.sub0, %41:vgpr_32, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %8:vgpr_32, killed %21:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+...
+
+---
+name: promote_async_load_offset_two_offsets
+machineFunctionInfo:
+ stackPtrOffsetReg: '$sgpr32'
+ frameOffsetReg: '$sgpr33'
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ liveins: $vgpr0, $sgpr0_sgpr1, $ttmp7
+ ; GFX1250-LABEL: name: promote_async_load_offset_two_offsets
+ ; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 516, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, dead $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 260, $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -260, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 4, $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -4, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %0:vgpr_32(s32) = COPY $vgpr0
+ %5:vgpr_32 = V_LSHLREV_B32_e64 8, %0:vgpr_32(s32), implicit $exec
+ %7:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
+ %8:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
+ %32:vgpr_32, %34:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 256, 0, implicit $exec
+ %33:vgpr_32, %35:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %34:sreg_32_xm0_xexec, 0, implicit $exec
+ %17:vreg_64_align2 = REG_SEQUENCE %32:vgpr_32, %subreg.sub0, %33:vgpr_32, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %7:vgpr_32, killed %17:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %40:vgpr_32, %36:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 512, 0, implicit $exec
+ %41:vgpr_32, %37:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %36:sreg_32_xm0_xexec, 0, implicit $exec
+ %21:vreg_64_align2 = REG_SEQUENCE %40:vgpr_32, %subreg.sub0, %41:vgpr_32, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 %8:vgpr_32, killed %21:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %42:vgpr_32, %38:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 516, 0, implicit $exec
+ %43:vgpr_32, %39:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %38:sreg_32_xm0_xexec, 0, implicit $exec
+ %22:vreg_64_align2 = REG_SEQUENCE %42:vgpr_32, %subreg.sub0, %43:vgpr_32, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %8:vgpr_32, killed %22:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+...
>From e418da050314bfedd1e78e024683ff69a9b61001 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Wed, 21 Jan 2026 02:37:07 -0800
Subject: [PATCH 2/4] Add V_ADD_U64 pattern
---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 121 +++++++++++++++++-
.../promote-constOffset-to-imm-gfx12.ll | 97 ++++++++++++++
.../promote-constOffset-to-imm-gfx12.mir | 95 +++++++++-----
3 files changed, 278 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index f50333b5886a7..3be1787fcb6c2 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -193,6 +193,8 @@ class SILoadStoreOptimizer {
unsigned LoSubReg = 0;
unsigned HiSubReg = 0;
+ // True when using V_ADD_U64_e64 pattern
+ bool UseV64Pattern = false;
};
struct MemAddress {
@@ -283,6 +285,9 @@ class SILoadStoreOptimizer {
Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
std::optional<int32_t> extractConstOffset(const MachineOperand &Op) const;
+ std::optional<int64_t> extractConstOffset64(const MachineOperand &Op) const;
+ bool processBaseWithConstOffset64(const MachineOperand &Base,
+ MemAddress &Addr) const;
void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
/// Promotes constant offset to the immediate by adjusting the base. It
/// tries to use a base from the nearby instructions that allows it to have
@@ -2166,6 +2171,33 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
MachineBasicBlock::iterator MBBI = MI.getIterator();
const DebugLoc &DL = MI.getDebugLoc();
+ LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
+
+ // Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
+ if (Addr.Base.UseV64Pattern) {
+ Register FullDestReg = MRI->createVirtualRegister(TRI->getVGPR64Class());
+
+ // Load the 64-bit offset into an SGPR pair if needed
+ Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
+ MachineInstr *MovOffset =
+ BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
+ OffsetReg)
+ .addImm(Addr.Offset);
+ (void)MovOffset;
+ LLVM_DEBUG(dbgs() << " "; MovOffset->dump(););
+
+ MachineInstr *Add64 =
+ BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U64_e64), FullDestReg)
+ .addReg(Addr.Base.LoReg)
+ .addReg(OffsetReg, RegState::Kill)
+ .addImm(0);
+ (void)Add64;
+ LLVM_DEBUG(dbgs() << " "; Add64->dump(); dbgs() << "\n";);
+
+ return FullDestReg;
+ }
+
+ // Original carry-chain pattern (V_ADD_CO_U32 + V_ADDC_U32)
assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 ||
Addr.Base.LoSubReg) &&
"Expected 32-bit Base-Register-Low!!");
@@ -2174,7 +2206,6 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
Addr.Base.HiSubReg) &&
"Expected 32-bit Base-Register-Hi!!");
- LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
MachineOperand OffsetLo = createRegOrImm(static_cast<int32_t>(Addr.Offset), MI);
MachineOperand OffsetHi =
createRegOrImm(static_cast<int32_t>(Addr.Offset >> 32), MI);
@@ -2243,6 +2274,72 @@ SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) const {
return Def->getOperand(1).getImm();
}
+std::optional<int64_t>
+SILoadStoreOptimizer::extractConstOffset64(const MachineOperand &Op) const {
+ if (Op.isImm())
+ return Op.getImm();
+
+ if (!Op.isReg())
+ return std::nullopt;
+
+ MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg());
+ if (!Def)
+ return std::nullopt;
+
+ // Handle S_MOV_B64_IMM_PSEUDO for 64-bit immediates
+ if (Def->getOpcode() == AMDGPU::S_MOV_B64_IMM_PSEUDO)
+ return TII->getNamedImmOperand(*Def, AMDGPU::OpName::src0);
+
+ return std::nullopt;
+}
+
+// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
+// Returns true if successful, populating Addr with base register info and
+// offset.
+bool SILoadStoreOptimizer::processBaseWithConstOffset64(
+ const MachineOperand &Base, MemAddress &Addr) const {
+ if (!Base.isReg())
+ return false;
+
+ MachineInstr *Def = MRI->getUniqueVRegDef(Base.getReg());
+ if (!Def || Def->getOpcode() != AMDGPU::V_ADD_U64_e64)
+ return false;
+
+ const auto *Src0 = TII->getNamedOperand(*Def, AMDGPU::OpName::src0);
+ const auto *Src1 = TII->getNamedOperand(*Def, AMDGPU::OpName::src1);
+ if (!Src0 || !Src1)
+ return false;
+
+ const MachineOperand *ConstOp = nullptr;
+ const MachineOperand *BaseOp = nullptr;
+
+ auto Offset = extractConstOffset64(*Src1);
+
+ if (Offset) {
+ ConstOp = Src1;
+ BaseOp = Src0;
+ Addr.Offset = *Offset;
+ } else {
+ // Both or neither are constants - can't handle this pattern
+ return false;
+ }
+
+ // Now extract the base register (which should be a 64-bit VGPR).
+ assert(BaseOp->isReg() && "Expected base operand to be a register");
+ MachineInstr *BaseDef = MRI->getUniqueVRegDef(BaseOp->getReg());
+ assert(BaseDef && "Expected definition for base register");
+ assert(TRI->getRegSizeInBits(BaseOp->getReg(), *MRI) == 64 &&
+ BaseOp->getSubReg() == 0 &&
+ "Expected 64-bit Base Register for V_ADD_U64_e64 pattern and no "
+ "subregisters");
+ Addr.Base.LoReg = BaseOp->getReg();
+ Addr.Base.HiReg = 0;
+ Addr.Base.LoSubReg = 0;
+ Addr.Base.HiSubReg = 0;
+ Addr.Base.UseV64Pattern = true;
+ return true;
+}
+
// Analyze Base and extracts:
// - 32bit base registers, subregisters
// - 64bit constant offset
@@ -2253,14 +2350,27 @@ SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) const {
// %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec
// %Base:vreg_64 =
// REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
+//
+// Also handles V_ADD_U64_e64 pattern (gfx1250+):
+// %OFFSET:sreg_64 = S_MOV_B64_IMM_PSEUDO 256
+// %Base:vreg_64 = V_ADD_U64_e64 %BASE:vreg_64, %OFFSET:sreg_64, 0
void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
MemAddress &Addr) const {
if (!Base.isReg())
return;
MachineInstr *Def = MRI->getUniqueVRegDef(Base.getReg());
- if (!Def || Def->getOpcode() != AMDGPU::REG_SEQUENCE
- || Def->getNumOperands() != 5)
+ if (!Def)
+ return;
+
+ // Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
+ if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
+ if (processBaseWithConstOffset64(Base, Addr))
+ return;
+ }
+
+ // Fall through to REG_SEQUENCE + V_ADD_CO_U32 + V_ADDC_U32 pattern
+ if (Def->getOpcode() != AMDGPU::REG_SEQUENCE || Def->getNumOperands() != 5)
return;
MachineOperand BaseLo = Def->getOperand(1);
@@ -2321,10 +2431,9 @@ void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
return;
Register OldVDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
- Register NewVDst =
- MRI->createVirtualRegister(TRI->getRegClassForReg(*MRI, OldVDst));
+ Register NewVDst = MRI->createVirtualRegister(MRI->getRegClass(OldVDst));
MachineBasicBlock &MBB = *MI.getParent();
- DebugLoc DL = MI.getDebugLoc();
+ const DebugLoc &DL = MI.getDebugLoc();
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), NewVDst)
.addReg(OldVDst)
.addImm(-OffsetDiff)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
new file mode 100644
index 0000000000000..f557318cffb1f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250 %s
+
+declare void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1), ptr addrspace(3), i32, i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+ at lds = addrspace(3) global [1024 x i8] poison, align 16
+
+; Test when there is a positive offset value
+define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %src) {
+; GFX1250-LABEL: promote_async_load_offset_negative:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v0, s[0:1]
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
+; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xfffffe00, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v[2:3], off offset:512
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep.offset = shl i32 %tid, 0
+ %lds.gep = getelementptr i8, ptr addrspace(3) @lds, i32 0
+
+ ; First load at base + 256
+ %offset0 = add i32 256, %gep.offset
+ %zext0 = zext i32 %offset0 to i64
+ %gep0 = getelementptr i8, ptr addrspace(1) %src, i64 %zext0
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep0, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ; Second load at base + 512 (+512 from 0)
+ %offset1 = add i32 512, %gep.offset
+ %zext1 = zext i32 %offset1 to i64
+ %gep1 = getelementptr i8, ptr addrspace(1) %src, i64 %zext1
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep1, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ; Final load at base + 0
+ %offset2 = add i32 0, %gep.offset
+ %zext2 = zext i32 %offset2 to i64
+ %gep2 = getelementptr i8, ptr addrspace(1) %src, i64 %zext2
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep2, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ret void
+}
+
+; Test when there is a negative offset value
+define amdgpu_kernel void @promote_async_load_offset_positive(ptr addrspace(1) %src) {
+; GFX1250-LABEL: promote_async_load_offset_positive:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v0, s[0:1]
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
+; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0x100, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 0x200, v[2:3]
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v[2:3], off offset:-256
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
+; GFX1250-NEXT: s_endpgm
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep.offset = shl i32 %tid, 0
+ %lds.gep = getelementptr i8, ptr addrspace(3) @lds, i32 0
+
+ ; First load at base + 0
+ %offset2 = add i32 0, %gep.offset
+ %zext2 = zext i32 %offset2 to i64
+ %gep2 = getelementptr i8, ptr addrspace(1) %src, i64 %zext2
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep2, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ; Second load at base + 256 (-256 from 512)
+ %offset0 = add i32 256, %gep.offset
+ %zext0 = zext i32 %offset0 to i64
+ %gep0 = getelementptr i8, ptr addrspace(1) %src, i64 %zext0
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep0, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ; Final load at base + 512
+ %offset1 = add i32 512, %gep.offset
+ %zext1 = zext i32 %offset1 to i64
+ %gep1 = getelementptr i8, ptr addrspace(1) %src, i64 %zext1
+ call void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gep1, ptr addrspace(3) %lds.gep, i32 0, i32 0)
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
index 3f243f39fa585..9215c0e664505 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -verify-machineinstrs -start-before=si-load-store-opt -stop-before=si-insert-waitcnts --verify-machineinstrs -o - %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -start-before=si-load-store-opt -stop-before=si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GFX1250 %s
---
name: promote_async_load_offset
@@ -20,18 +20,18 @@ body: |
; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 256, $vgpr1, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- %0:vgpr_32(s32) = COPY $vgpr0
- %5:vgpr_32 = V_LSHLREV_B32_e64 8, %0:vgpr_32(s32), implicit $exec
- %7:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
- %8:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
- %32:vgpr_32, %34:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 256, 0, implicit $exec
- %33:vgpr_32, %35:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %34:sreg_32_xm0_xexec, 0, implicit $exec
- %17:vreg_64_align2 = REG_SEQUENCE %32:vgpr_32, %subreg.sub0, %33:vgpr_32, %subreg.sub1
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %7:vgpr_32, killed %17:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- %40:vgpr_32, %36:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 512, 0, implicit $exec
- %41:vgpr_32, %37:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %36:sreg_32_xm0_xexec, 0, implicit $exec
- %21:vreg_64_align2 = REG_SEQUENCE %40:vgpr_32, %subreg.sub0, %41:vgpr_32, %subreg.sub1
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %8:vgpr_32, killed %21:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
+ %2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %3:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %4:vgpr_32, %5:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 256, 0, implicit $exec
+ %6:vgpr_32, %7:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %5, 0, implicit $exec
+ %8:vreg_64_align2 = REG_SEQUENCE %4, %subreg.sub0, %6, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %2, killed %8, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %9:vgpr_32, %10:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 512, 0, implicit $exec
+ %11:vgpr_32, %12:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %10, 0, implicit $exec
+ %13:vreg_64_align2 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %13, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
...
---
@@ -55,20 +55,57 @@ body: |
; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 4, $vgpr1, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -4, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- %0:vgpr_32(s32) = COPY $vgpr0
- %5:vgpr_32 = V_LSHLREV_B32_e64 8, %0:vgpr_32(s32), implicit $exec
- %7:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
- %8:vgpr_32 = disjoint V_OR_B32_e64 %5:vgpr_32, 0, implicit $exec
- %32:vgpr_32, %34:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 256, 0, implicit $exec
- %33:vgpr_32, %35:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %34:sreg_32_xm0_xexec, 0, implicit $exec
- %17:vreg_64_align2 = REG_SEQUENCE %32:vgpr_32, %subreg.sub0, %33:vgpr_32, %subreg.sub1
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %7:vgpr_32, killed %17:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- %40:vgpr_32, %36:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 512, 0, implicit $exec
- %41:vgpr_32, %37:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %36:sreg_32_xm0_xexec, 0, implicit $exec
- %21:vreg_64_align2 = REG_SEQUENCE %40:vgpr_32, %subreg.sub0, %41:vgpr_32, %subreg.sub1
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 %8:vgpr_32, killed %21:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- %42:vgpr_32, %38:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0:vgpr_32, 516, 0, implicit $exec
- %43:vgpr_32, %39:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0:vgpr_32, 0, killed %38:sreg_32_xm0_xexec, 0, implicit $exec
- %22:vreg_64_align2 = REG_SEQUENCE %42:vgpr_32, %subreg.sub0, %43:vgpr_32, %subreg.sub1
- GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %8:vgpr_32, killed %22:vreg_64_align2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
+ %2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %3:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %4:vgpr_32, %5:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 256, 0, implicit $exec
+ %6:vgpr_32, %7:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %5, 0, implicit $exec
+ %8:vreg_64_align2 = REG_SEQUENCE %4, %subreg.sub0, %6, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %2, killed %8, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %9:vgpr_32, %10:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 512, 0, implicit $exec
+ %11:vgpr_32, %12:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %10, 0, implicit $exec
+ %13:vreg_64_align2 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 %3, killed %13, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %14:vgpr_32, %15:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 516, 0, implicit $exec
+ %16:vgpr_32, %17:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %15, 0, implicit $exec
+ %18:vreg_64_align2 = REG_SEQUENCE %14, %subreg.sub0, %16, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %18, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+...
+
+---
+name: promote_async_load_u64
+machineFunctionInfo:
+ stackPtrOffsetReg: '$sgpr32'
+ frameOffsetReg: '$sgpr33'
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX1250-LABEL: name: promote_async_load_u64
+ ; GFX1250: liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; GFX1250-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr0, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR $vgpr1, $sgpr0_sgpr1, $vgpr0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 killed $sgpr0_sgpr1, $vgpr0_vgpr1, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e64 256, 0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 512, killed $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -256, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ %3:sgpr_64 = COPY $sgpr4_sgpr5
+ %0:vgpr_32 = COPY $vgpr0
+ early-clobber %5:sreg_64_xexec = S_LOAD_DWORDX2_IMM_ec %3, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %8:vgpr_32 = V_AND_B32_e64 %0, 1023, implicit $exec
+ %9:vreg_64_align2 = REG_SEQUENCE %8, %subreg.sub0, %6, %subreg.sub1
+ %10:vreg_64_align2 = V_ADD_U64_e64 %5, killed %9, 0, implicit $exec
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR %6, %5, %8, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %13:vreg_64_align2 = V_ADD_U64_e64 %10, 256, 0, implicit $exec
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 %6, killed %13, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %15:vreg_64_align2 = V_ADD_U64_e64 %10, 512, 0, implicit $exec
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 %6, killed %15, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ S_ENDPGM 0
...
>From 8f1490581621c7be420c3af59b5929516e687522 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Wed, 21 Jan 2026 12:42:10 -0800
Subject: [PATCH 3/4] Refactor to use getImmOrMaterializedImm
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 30 +----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 18 +++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 103 +++++-------------
4 files changed, 56 insertions(+), 97 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 2df9267cde1f2..99353a9ce2967 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -242,7 +242,6 @@ class SIFoldOperandsImpl {
SmallVectorImpl<FoldCandidate> &FoldList,
SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
- std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
bool tryConstantFoldOp(MachineInstr *MI) const;
bool tryFoldCndMask(MachineInstr &MI) const;
bool tryFoldZeroHighBits(MachineInstr &MI) const;
@@ -1567,24 +1566,6 @@ static unsigned getMovOpc(bool IsScalar) {
return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
}
-std::optional<int64_t>
-SIFoldOperandsImpl::getImmOrMaterializedImm(MachineOperand &Op) const {
- if (Op.isImm())
- return Op.getImm();
-
- if (!Op.isReg() || !Op.getReg().isVirtual())
- return std::nullopt;
-
- const MachineInstr *Def = MRI->getVRegDef(Op.getReg());
- if (Def && Def->isMoveImmediate()) {
- const MachineOperand &ImmSrc = Def->getOperand(1);
- if (ImmSrc.isImm())
- return TII->extractSubregFromImm(ImmSrc.getImm(), Op.getSubReg());
- }
-
- return std::nullopt;
-}
-
// Try to simplify operations with a constant that may appear after instruction
// selection.
// TODO: See if a frame index with a fixed offset can fold.
@@ -1599,7 +1580,7 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
return false;
MachineOperand *Src0 = &MI->getOperand(Src0Idx);
- std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(*Src0);
+ std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(*Src0);
if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
Opc == AMDGPU::S_NOT_B32) &&
@@ -1615,7 +1596,7 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
return false;
MachineOperand *Src1 = &MI->getOperand(Src1Idx);
- std::optional<int64_t> Src1Imm = getImmOrMaterializedImm(*Src1);
+ std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(*Src1);
if (!Src0Imm && !Src1Imm)
return false;
@@ -1706,11 +1687,11 @@ bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
if (!Src1->isIdenticalTo(*Src0)) {
- std::optional<int64_t> Src1Imm = getImmOrMaterializedImm(*Src1);
+ std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(*Src1);
if (!Src1Imm)
return false;
- std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(*Src0);
+ std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(*Src0);
if (!Src0Imm || *Src0Imm != *Src1Imm)
return false;
}
@@ -1744,7 +1725,8 @@ bool SIFoldOperandsImpl::tryFoldZeroHighBits(MachineInstr &MI) const {
MI.getOpcode() != AMDGPU::V_AND_B32_e32)
return false;
- std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(MI.getOperand(1));
+ std::optional<int64_t> Src0Imm =
+ TII->getImmOrMaterializedImm(MI.getOperand(1));
if (!Src0Imm || *Src0Imm != 0xffff || !MI.getOperand(2).isReg())
return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6392022368785..c9ce04a9d5d26 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1372,6 +1372,24 @@ bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
}
}
+std::optional<int64_t>
+SIInstrInfo::getImmOrMaterializedImm(MachineOperand &Op) const {
+ if (Op.isImm())
+ return Op.getImm();
+
+ if (!Op.isReg() || !Op.getReg().isVirtual())
+ return std::nullopt;
+ MachineRegisterInfo &MRI = Op.getParent()->getMF()->getRegInfo();
+ const MachineInstr *Def = MRI.getVRegDef(Op.getReg());
+ if (Def && Def->isMoveImmediate()) {
+ const MachineOperand &ImmSrc = Def->getOperand(1);
+ if (ImmSrc.isImm())
+ return extractSubregFromImm(ImmSrc.getImm(), Op.getSubReg());
+ }
+
+ return std::nullopt;
+}
+
unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
if (RI.isAGPRClass(DstRC))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 44037d01dfdd1..17a21add241c7 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -303,6 +303,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
int64_t &ImmVal) const override;
+ std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
+
unsigned getVectorRegSpillSaveOpcode(Register Reg,
const TargetRegisterClass *RC,
unsigned Size,
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 3be1787fcb6c2..e0db71a22b1a7 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -284,8 +284,6 @@ class SILoadStoreOptimizer {
void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
- std::optional<int32_t> extractConstOffset(const MachineOperand &Op) const;
- std::optional<int64_t> extractConstOffset64(const MachineOperand &Op) const;
bool processBaseWithConstOffset64(const MachineOperand &Base,
MemAddress &Addr) const;
void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
@@ -2175,7 +2173,8 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
// Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
if (Addr.Base.UseV64Pattern) {
- Register FullDestReg = MRI->createVirtualRegister(TRI->getVGPR64Class());
+ Register FullDestReg = MRI->createVirtualRegister(
+ TII->getRegClass(TII->get(AMDGPU::V_ADD_U64_e64), 0));
// Load the 64-bit offset into an SGPR pair if needed
Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
@@ -2183,16 +2182,15 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
OffsetReg)
.addImm(Addr.Offset);
- (void)MovOffset;
- LLVM_DEBUG(dbgs() << " "; MovOffset->dump(););
-
MachineInstr *Add64 =
BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U64_e64), FullDestReg)
.addReg(Addr.Base.LoReg)
.addReg(OffsetReg, RegState::Kill)
.addImm(0);
+ (void)MovOffset;
(void)Add64;
- LLVM_DEBUG(dbgs() << " "; Add64->dump(); dbgs() << "\n";);
+ LLVM_DEBUG(dbgs() << " " << *MovOffset << "\n";
+ dbgs() << " " << *Add64 << "\n\n";);
return FullDestReg;
}
@@ -2217,23 +2215,19 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
Register DestSub0 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register DestSub1 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
MachineInstr *LoHalf =
- BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), DestSub0)
- .addReg(CarryReg, RegState::Define)
- .addReg(Addr.Base.LoReg, 0, Addr.Base.LoSubReg)
- .add(OffsetLo)
- .addImm(0); // clamp bit
- (void)LoHalf;
- LLVM_DEBUG(dbgs() << " "; LoHalf->dump(););
+ BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), DestSub0)
+ .addReg(CarryReg, RegState::Define)
+ .addReg(Addr.Base.LoReg, 0, Addr.Base.LoSubReg)
+ .add(OffsetLo)
+ .addImm(0); // clamp bit
MachineInstr *HiHalf =
- BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), DestSub1)
- .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
- .addReg(Addr.Base.HiReg, 0, Addr.Base.HiSubReg)
- .add(OffsetHi)
- .addReg(CarryReg, RegState::Kill)
- .addImm(0); // clamp bit
- (void)HiHalf;
- LLVM_DEBUG(dbgs() << " "; HiHalf->dump(););
+ BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), DestSub1)
+ .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
+ .addReg(Addr.Base.HiReg, 0, Addr.Base.HiSubReg)
+ .add(OffsetHi)
+ .addReg(CarryReg, RegState::Kill)
+ .addImm(0); // clamp bit
Register FullDestReg = MRI->createVirtualRegister(TRI->getVGPR64Class());
MachineInstr *FullBase =
@@ -2242,8 +2236,13 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
.addImm(AMDGPU::sub0)
.addReg(DestSub1)
.addImm(AMDGPU::sub1);
+
+ (void)LoHalf;
+ (void)HiHalf;
(void)FullBase;
- LLVM_DEBUG(dbgs() << " "; FullBase->dump(); dbgs() << "\n";);
+ LLVM_DEBUG(dbgs() << " " << *LoHalf << "\n";
+ dbgs() << " " << *HiHalf << "\n";
+ dbgs() << " " << *FullBase << "\n\n";);
return FullDestReg;
}
@@ -2258,41 +2257,6 @@ void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset);
}
-std::optional<int32_t>
-SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) const {
- if (Op.isImm())
- return Op.getImm();
-
- if (!Op.isReg())
- return std::nullopt;
-
- MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg());
- if (!Def || Def->getOpcode() != AMDGPU::S_MOV_B32 ||
- !Def->getOperand(1).isImm())
- return std::nullopt;
-
- return Def->getOperand(1).getImm();
-}
-
-std::optional<int64_t>
-SILoadStoreOptimizer::extractConstOffset64(const MachineOperand &Op) const {
- if (Op.isImm())
- return Op.getImm();
-
- if (!Op.isReg())
- return std::nullopt;
-
- MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg());
- if (!Def)
- return std::nullopt;
-
- // Handle S_MOV_B64_IMM_PSEUDO for 64-bit immediates
- if (Def->getOpcode() == AMDGPU::S_MOV_B64_IMM_PSEUDO)
- return TII->getNamedImmOperand(*Def, AMDGPU::OpName::src0);
-
- return std::nullopt;
-}
-
// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
// Returns true if successful, populating Addr with base register info and
// offset.
@@ -2301,22 +2265,18 @@ bool SILoadStoreOptimizer::processBaseWithConstOffset64(
if (!Base.isReg())
return false;
- MachineInstr *Def = MRI->getUniqueVRegDef(Base.getReg());
+ MachineInstr *Def = MRI->getVRegDef(Base.getReg());
if (!Def || Def->getOpcode() != AMDGPU::V_ADD_U64_e64)
return false;
- const auto *Src0 = TII->getNamedOperand(*Def, AMDGPU::OpName::src0);
- const auto *Src1 = TII->getNamedOperand(*Def, AMDGPU::OpName::src1);
- if (!Src0 || !Src1)
- return false;
+ MachineOperand *Src0 = TII->getNamedOperand(*Def, AMDGPU::OpName::src0);
+ MachineOperand *Src1 = TII->getNamedOperand(*Def, AMDGPU::OpName::src1);
- const MachineOperand *ConstOp = nullptr;
const MachineOperand *BaseOp = nullptr;
- auto Offset = extractConstOffset64(*Src1);
+ auto Offset = TII->getImmOrMaterializedImm(*Src1);
if (Offset) {
- ConstOp = Src1;
BaseOp = Src0;
Addr.Offset = *Offset;
} else {
@@ -2333,9 +2293,6 @@ bool SILoadStoreOptimizer::processBaseWithConstOffset64(
"Expected 64-bit Base Register for V_ADD_U64_e64 pattern and no "
"subregisters");
Addr.Base.LoReg = BaseOp->getReg();
- Addr.Base.HiReg = 0;
- Addr.Base.LoSubReg = 0;
- Addr.Base.HiSubReg = 0;
Addr.Base.UseV64Pattern = true;
return true;
}
@@ -2385,14 +2342,14 @@ void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base
!BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64)
return;
- const auto *Src0 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src0);
- const auto *Src1 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src1);
+ MachineOperand *Src0 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src0);
+ MachineOperand *Src1 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src1);
- auto Offset0P = extractConstOffset(*Src0);
+ auto Offset0P = TII->getImmOrMaterializedImm(*Src0);
if (Offset0P)
BaseLo = *Src1;
else {
- if (!(Offset0P = extractConstOffset(*Src1)))
+ if (!(Offset0P = TII->getImmOrMaterializedImm(*Src1)))
return;
BaseLo = *Src0;
}
>From 72c9cb9ed817a25cf6b9d40094a0f75247a7622d Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <Ryan.Mitchell at amd.com>
Date: Thu, 22 Jan 2026 07:58:45 -0800
Subject: [PATCH 4/4] Address review
---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 22 ++++++-------------
1 file changed, 7 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index e0db71a22b1a7..93dba29aa6c0c 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -284,7 +284,8 @@ class SILoadStoreOptimizer {
void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
- bool processBaseWithConstOffset64(const MachineOperand &Base,
+ bool processBaseWithConstOffset64(MachineInstr *AddDef,
+ const MachineOperand &Base,
MemAddress &Addr) const;
void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
/// Promotes constant offset to the immediate by adjusting the base. It
@@ -2261,16 +2262,12 @@ void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
// Returns true if successful, populating Addr with base register info and
// offset.
bool SILoadStoreOptimizer::processBaseWithConstOffset64(
- const MachineOperand &Base, MemAddress &Addr) const {
+ MachineInstr *AddDef, const MachineOperand &Base, MemAddress &Addr) const {
if (!Base.isReg())
return false;
- MachineInstr *Def = MRI->getVRegDef(Base.getReg());
- if (!Def || Def->getOpcode() != AMDGPU::V_ADD_U64_e64)
- return false;
-
- MachineOperand *Src0 = TII->getNamedOperand(*Def, AMDGPU::OpName::src0);
- MachineOperand *Src1 = TII->getNamedOperand(*Def, AMDGPU::OpName::src1);
+ MachineOperand *Src0 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src0);
+ MachineOperand *Src1 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src1);
const MachineOperand *BaseOp = nullptr;
@@ -2285,13 +2282,8 @@ bool SILoadStoreOptimizer::processBaseWithConstOffset64(
}
// Now extract the base register (which should be a 64-bit VGPR).
- assert(BaseOp->isReg() && "Expected base operand to be a register");
- MachineInstr *BaseDef = MRI->getUniqueVRegDef(BaseOp->getReg());
+ MachineInstr *BaseDef = MRI->getVRegDef(BaseOp->getReg());
assert(BaseDef && "Expected definition for base register");
- assert(TRI->getRegSizeInBits(BaseOp->getReg(), *MRI) == 64 &&
- BaseOp->getSubReg() == 0 &&
- "Expected 64-bit Base Register for V_ADD_U64_e64 pattern and no "
- "subregisters");
Addr.Base.LoReg = BaseOp->getReg();
Addr.Base.UseV64Pattern = true;
return true;
@@ -2322,7 +2314,7 @@ void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base
// Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
- if (processBaseWithConstOffset64(Base, Addr))
+ if (processBaseWithConstOffset64(Def, Base, Addr))
return;
}
More information about the llvm-commits
mailing list