[llvm] [MachineLateInstrsCleanup] Reuse redundant spill-slot reloads (PR #220180)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 00:12:45 PDT 2026
https://github.com/anupkum-sgs updated https://github.com/llvm/llvm-project/pull/220180
>From 748a2aae267a216abf7fe1ff1bc78f26653cce11 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Tue, 1 Sep 2026 11:21:55 +0530
Subject: [PATCH 1/8] [MachineLateInstrsCleanup] Add tests for reuse of
invariant loads
The pass removes a definition when an identical one is already available in
the same register. isCandidate() calls isSafeToMove() with SawStore set, which
rejects an ordinary load but still admits an invariant one, because
isSafeToMove() only consults SawStore for a load that
isDereferenceableInvariantLoad() does not accept. So the pass already removes
a duplicate constant pool load, and a duplicate load from an immutable frame
object. Nothing tested that.
Add MIR tests for both, and for two cases that must not be reused: a load from
a mutable frame object, and a constant pool load whose register is redefined
in between.
The tests use a 32-bit target because a constant pool entry is addressed
absolutely there. On x86-64 the load goes through $rip, and isCandidate() only
lets an instruction use the frame register, so that form is never tracked.
Test only, no functional change.
---
.../machine-latecleanup-invariant-loads.mir | 139 ++++++++++++++++++
1 file changed, 139 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
new file mode 100644
index 00000000000000..d77d64f2582b25
--- /dev/null
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
@@ -0,0 +1,139 @@
+# RUN: llc -mtriple=i686-unknown-linux-gnu -run-pass=machine-latecleanup %s \
+# RUN: -o - -verify-machineinstrs | FileCheck %s
+#
+# An invariant load always returns the same value, so the pass can reuse an
+# earlier one even when a store or a call comes in between.
+#
+# These tests use a 32-bit target on purpose. There a constant pool entry is
+# addressed absolutely, while on x86-64 the load goes through $rip. isCandidate()
+# only lets an instruction use the frame register, so the x86-64 form is never
+# tracked and there would be nothing to test.
+
+--- |
+ define void @constant_pool_load_is_reused() #0 {
+ ret void
+ }
+
+ define void @immutable_argument_load_is_reused() #0 {
+ ret void
+ }
+
+ define void @mutable_argument_load_is_not_reused() #0 {
+ ret void
+ }
+
+ define void @constant_pool_load_not_reused_after_clobber() #0 {
+ ret void
+ }
+
+ declare void @sink()
+
+ attributes #0 = { "frame-pointer"="all" }
+...
+---
+# A store cannot change a constant pool entry.
+name: constant_pool_load_is_reused
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 16
+ maxAlignment: 4
+constants:
+ - id: 0
+ value: 'i32 85'
+ alignment: 4
+stack:
+ - { id: 0, type: spill-slot, offset: -8, size: 4, alignment: 4 }
+body: |
+ bb.0:
+ liveins: $ebp, $ecx
+
+ ; CHECK-LABEL: name: constant_pool_load_is_reused
+ ; CHECK: $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ ; CHECK-NEXT: MOV32mr $ebp, 1, $noreg, -8, $noreg, $ecx :: (store (s32) into %stack.0)
+ ; CHECK-NEXT: MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ ; CHECK-NEXT: RET32
+ $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ MOV32mr $ebp, 1, $noreg, -8, $noreg, $ecx :: (store (s32) into %stack.0)
+ $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ RET32
+...
+---
+# An incoming argument slot is immutable, so not even a call can change it.
+# $ebx is callee-saved, so the call's register mask leaves the tracked
+# definition alone and only the memory reasoning is under test.
+name: immutable_argument_load_is_reused
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 16
+ maxAlignment: 4
+ hasCalls: true
+fixedStack:
+ - { id: 0, offset: 8, size: 4, alignment: 4, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $ebp
+
+ ; CHECK-LABEL: name: immutable_argument_load_is_reused
+ ; CHECK: $ebx = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: CALLpcrel32
+ ; CHECK-NEXT: MOV32mr $ebp, 1, $noreg, -12, $noreg, $ebx
+ ; CHECK-NEXT: RET32
+ $ebx = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ CALLpcrel32 target-flags(x86-plt) @sink, csr_32, implicit $esp, implicit $ssp, implicit-def $esp, implicit-def $ssp
+ $ebx = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ MOV32mr $ebp, 1, $noreg, -12, $noreg, $ebx
+ RET32
+...
+---
+# A mutable frame object is not invariant, so the pass has to leave it alone.
+name: mutable_argument_load_is_not_reused
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 16
+ maxAlignment: 4
+fixedStack:
+ - { id: 0, offset: 8, size: 4, alignment: 4, isImmutable: false }
+body: |
+ bb.0:
+ liveins: $ebp
+
+ ; CHECK-LABEL: name: mutable_argument_load_is_not_reused
+ ; CHECK: $eax = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: $ecx = MOV32ri 1
+ ; CHECK-NEXT: $eax = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ ; CHECK-NEXT: RET32
+ $eax = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ $ecx = MOV32ri 1
+ $eax = MOV32rm $ebp, 1, $noreg, 8, $noreg :: (load (s32) from %fixed-stack.0)
+ MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ RET32
+...
+---
+# Redefining the loaded register still ends the reuse.
+name: constant_pool_load_not_reused_after_clobber
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 16
+ maxAlignment: 4
+constants:
+ - id: 0
+ value: 'i32 85'
+ alignment: 4
+body: |
+ bb.0:
+ liveins: $ebp
+
+ ; CHECK-LABEL: name: constant_pool_load_not_reused_after_clobber
+ ; CHECK: $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ ; CHECK-NEXT: $eax = MOV32ri 7
+ ; CHECK-NEXT: $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ ; CHECK-NEXT: MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ ; CHECK-NEXT: RET32
+ $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ $eax = MOV32ri 7
+ $eax = MOV32rm $noreg, 1, $noreg, %const.0, $noreg :: (load (s32) from constant-pool)
+ MOV32mr $ebp, 1, $noreg, -12, $noreg, $eax
+ RET32
+...
>From 6dc51ae2718e8557fa37f6e2714803d8164a06c5 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Tue, 1 Sep 2026 12:07:38 +0530
Subject: [PATCH 2/8] [MachineLateInstrsCleanup] Reuse redundant spill-slot
reloads
A reload of a spill slot is never reused today, because isCandidate() calls
isSafeToMove() with SawStore set, and that rejects every load which is not
invariant. An identical reload is therefore left in place even where the slot
provably has not been written since the earlier one, which costs a load in
spill-heavy code.
Track spill-slot reloads as well. isCandidate() now clears SawStore for a load
that goes through a single spill-slot memory operand, which lets isSafeToMove()
accept it. A volatile or atomic access is still refused, because isSafeToMove()
rejects those up front without ever looking at SawStore, and every other load
is treated exactly as before. processBlock() then takes responsibility for the
reload, and drops it as soon as it sees an instruction that may write the slot
it reads.
Limiting this to register-allocation spill slots is what makes the reasoning
sound. Such a slot is created by the register allocator and its address never
reaches the IR, so a store to a different spill slot cannot change what the
reload reads. That is what allows a reload to be reused across an unrelated
spill.
Everything else drops the tracked reload: a call, an instruction with unmodeled
side effects, and any store whose memory operand does not name a spill slot the
reload can be compared against.
MachineInstr::mayAlias() is deliberately not used for this. It reports
may-alias for any two distinct pseudo source values, so it cannot tell one
spill slot from another, and would defeat the case above.
Five autogenerated tests are regenerated: six removed reloads, plus one copy
that X86FixupBWInsts no longer widens, because removing a reload leaves the
reused register live where it previously was not.
---
llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp | 72 ++++++-
.../AArch64/fp-maximumnum-minimumnum.ll | 2 -
.../CodeGen/AArch64/vecreduce-fmaximumnum.ll | 1 -
.../CodeGen/AArch64/vecreduce-fminimumnum.ll | 1 -
.../machine-latecleanup-invariant-loads.mir | 8 +-
.../X86/machine-latecleanup-spill-reloads.mir | 196 ++++++++++++++++++
.../vector-interleaved-load-i16-stride-7.ll | 1 -
...lar-shift-by-byte-multiple-legalization.ll | 3 +-
8 files changed, 267 insertions(+), 17 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index 811cc4fe65f3f9..1d3a2248882637 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -11,6 +11,10 @@
// the result of rematerialization, while the addresses are redundant frame
// addressing anchor points created during Frame Indices elimination.
//
+// Reloads of spill slots are handled as well: these slots do not alias IR
+// values, so an earlier reload can be reused as long as nothing in between may
+// have written to that same slot.
+//
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
@@ -18,10 +22,13 @@
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -168,15 +175,61 @@ void MachineLateInstrsCleanup::removeRedundantDef(MachineInstr *MI) {
++NumRemoved;
}
+// Return true if MI accesses a register-allocation spill slot through a single
+// memory operand, and if so also the frame index of that slot in FI.
+static bool isSingleSpillSlotAccess(const MachineInstr &MI,
+ const MachineFrameInfo &MFI, int &FI) {
+ if (!MI.hasOneMemOperand())
+ return false;
+ const MachineMemOperand *MMO = *MI.memoperands_begin();
+ const auto *PSV =
+ dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->getPseudoValue());
+ if (!PSV)
+ return false;
+ FI = PSV->getFrameIndex();
+ return MFI.isSpillSlotObjectIndex(FI);
+}
+
+// Return true if MI may write the memory read by Reload, so that Reload can no
+// longer be reused. The only tracked loads are invariant ones and spill-slot
+// reloads, so a store to a different spill slot cannot change the loaded value.
+static bool mayOverwriteReload(const MachineInstr &MI,
+ const MachineInstr &Reload,
+ const MachineFrameInfo &MFI) {
+ // Invariant loads, such as constant pool entries and immutable frame
+ // objects, keep reading the same value whatever memory MI writes.
+ if (!Reload.mayLoad() || Reload.isDereferenceableInvariantLoad())
+ return false;
+ if (!MI.mayStore() && !MI.isCall() && !MI.hasUnmodeledSideEffects())
+ return false;
+ int StoreFI, ReloadFI;
+ if (isSingleSpillSlotAccess(MI, MFI, StoreFI) &&
+ isSingleSpillSlotAccess(Reload, MFI, ReloadFI))
+ return StoreFI == ReloadFI;
+ // Anything else may reach the slot, e.g. a call or a store through a
+ // pointer to an escaped stack object.
+ return true;
+}
+
// Return true if MI is a potential candidate for reuse/removal and if so
// also the register it defines in DefedReg. A candidate is a simple
-// instruction that does not touch memory, has only one register definition
-// and the only reg it may use is FrameReg. Typically this is an immediate
-// load or a load-address instruction.
+// instruction that has only one register definition and the only reg it may
+// use is FrameReg. Typically this is an immediate load or a load-address
+// instruction. The only memory it may read is invariant memory or a spill slot
+// it reloads, since processBlock() keeps track of stores that may write the
+// slot.
static bool isCandidate(const MachineInstr *MI, Register &DefedReg,
- Register FrameReg) {
+ Register FrameReg, const MachineFrameInfo &MFI) {
DefedReg = MCRegister::NoRegister;
- bool SawStore = true;
+ // SawStore makes isSafeToMove() reject every load that is not invariant, on
+ // the grounds that a store may have changed what it reads. That is too blunt
+ // for a spill-slot reload, where only a write to the same slot matters and
+ // processBlock() already drops the reload as soon as it sees one. Clear
+ // SawStore for those reloads to defer to that finer check; every other load
+ // is left to isSafeToMove(), which admits the invariant ones and rejects the
+ // volatile and atomic accesses whatever SawStore says.
+ int FI;
+ bool SawStore = !(MI->mayLoad() && isSingleSpillSlotAccess(*MI, MFI, FI));
if (!MI->isSafeToMove(SawStore) || MI->isImplicitDef() || MI->isInlineAsm())
return false;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -220,6 +273,7 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
// Process MBB.
MachineFunction *MF = MBB->getParent();
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+ const MachineFrameInfo &MFI = MF->getFrameInfo();
Register FrameReg = TRI->getFrameRegister(*MF);
for (MachineInstr &MI : llvm::make_early_inc_range(*MBB)) {
// If FrameReg is modified, no previous load-address instructions (using
@@ -231,7 +285,7 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
}
Register DefedReg;
- bool IsCandidate = isCandidate(&MI, DefedReg, FrameReg);
+ bool IsCandidate = isCandidate(&MI, DefedReg, FrameReg, MFI);
// Check for an earlier identical and reusable instruction.
if (IsCandidate && MBBDefs.hasIdentical(DefedReg, &MI)) {
@@ -242,10 +296,12 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
continue;
}
- // Clear any entries in map that MI clobbers.
+ // Clear any entries in map that MI clobbers, either by redefining the
+ // register or by writing the stack slot a reload reads.
MBBDefs.remove_if([&](const auto &Entry) {
Register Reg = Entry.first;
- if (MI.modifiesRegister(Reg, TRI)) {
+ if (MI.modifiesRegister(Reg, TRI) ||
+ mayOverwriteReload(MI, *Entry.second, MFI)) {
MBBKills.erase(Reg);
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
index a457fe01d309d8..79c005c4ab274d 100644
--- a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
+++ b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
@@ -1236,7 +1236,6 @@ define fp128 @max_fp128(fp128 %x, fp128 %y) {
; CHECK-NEXT: // %bb.3: // %start
; CHECK-NEXT: mov v1.16b, v0.16b
; CHECK-NEXT: .LBB32_4: // %start
-; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
; CHECK-NEXT: str q1, [sp, #16] // 16-byte Spill
; CHECK-NEXT: bl __gttf2
; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
@@ -1879,7 +1878,6 @@ define fp128 @min_fp128(fp128 %x, fp128 %y) {
; CHECK-NEXT: // %bb.3: // %start
; CHECK-NEXT: mov v1.16b, v0.16b
; CHECK-NEXT: .LBB49_4: // %start
-; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
; CHECK-NEXT: str q1, [sp, #16] // 16-byte Spill
; CHECK-NEXT: bl __gttf2
; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmaximumnum.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmaximumnum.ll
index 75468cb45a2ade..ab7ed2b9ce89f6 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmaximumnum.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmaximumnum.ll
@@ -546,7 +546,6 @@ define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
; CHECK-NEXT: // %bb.3:
; CHECK-NEXT: mov v1.16b, v0.16b
; CHECK-NEXT: .LBB16_4:
-; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
; CHECK-NEXT: str q1, [sp, #16] // 16-byte Spill
; CHECK-NEXT: bl __gttf2
; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fminimumnum.ll b/llvm/test/CodeGen/AArch64/vecreduce-fminimumnum.ll
index bbbefffc06758a..6fb6b5ef36b052 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fminimumnum.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fminimumnum.ll
@@ -546,7 +546,6 @@ define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
; CHECK-NEXT: // %bb.3:
; CHECK-NEXT: mov v1.16b, v0.16b
; CHECK-NEXT: .LBB16_4:
-; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
; CHECK-NEXT: str q1, [sp, #16] // 16-byte Spill
; CHECK-NEXT: bl __lttf2
; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
index d77d64f2582b25..5c080481096d09 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
@@ -2,7 +2,10 @@
# RUN: -o - -verify-machineinstrs | FileCheck %s
#
# An invariant load always returns the same value, so the pass can reuse an
-# earlier one even when a store or a call comes in between.
+# earlier one even when a store or a call comes in between. That has to keep
+# working now that a spill-slot reload is tracked as well, because the stores
+# and calls which end the reuse of a reload must not end the reuse of an
+# invariant load.
#
# These tests use a 32-bit target on purpose. There a constant pool entry is
# addressed absolutely, while on x86-64 the load goes through $rip. isCandidate()
@@ -86,7 +89,8 @@ body: |
RET32
...
---
-# A mutable frame object is not invariant, so the pass has to leave it alone.
+# A mutable frame object is neither invariant nor a spill slot, so the pass has
+# to leave it alone.
name: mutable_argument_load_is_not_reused
tracksRegLiveness: true
frameInfo:
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
new file mode 100644
index 00000000000000..4265c32deed012
--- /dev/null
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
@@ -0,0 +1,196 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=machine-latecleanup %s \
+# RUN: -o - -verify-machineinstrs | FileCheck %s
+
+--- |
+ define void @remove_redundant_spill_reload() {
+ ret void
+ }
+
+ define void @same_spill_slot_store_blocks_reload_reuse() {
+ ret void
+ }
+
+ define void @different_spill_slot_store_keeps_reload_reusable() {
+ ret void
+ }
+
+ define void @non_spill_stack_slot_reload_is_not_reused() {
+ ret void
+ }
+
+ define void @call_blocks_reload_reuse() #0 {
+ ret void
+ }
+
+ define void @reload_reused_from_predecessors() {
+ ret void
+ }
+
+ declare void @sink()
+
+ attributes #0 = { "frame-pointer"="all" }
+...
+---
+name: remove_redundant_spill_reload
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: remove_redundant_spill_reload
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: $rcx = MOV64ri 1
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ $rcx = MOV64ri 1
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+name: same_spill_slot_store_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rcx
+
+ ; CHECK-LABEL: name: same_spill_slot_store_blocks_reload_reuse
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 32, $noreg, $rcx :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 32, $noreg, $rcx :: (store (s64) into %stack.0)
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+name: different_spill_slot_store_keeps_reload_reusable
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 56
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+ - { id: 2, type: spill-slot, offset: -32, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rcx
+
+ ; CHECK-LABEL: name: different_spill_slot_store_keeps_reload_reusable
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rcx :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 48, $noreg, $rax :: (store (s64) into %stack.2)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rcx :: (store (s64) into %stack.1)
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 48, $noreg, $rax :: (store (s64) into %stack.2)
+ RET64
+...
+---
+name: non_spill_stack_slot_reload_is_not_reused
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: non_spill_stack_slot_reload_is_not_reused
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: $rcx = MOV64ri 1
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ $rcx = MOV64ri 1
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+# The reload target is callee-saved and the frame register is $rbp, so neither
+# the call's regmask nor its implicit-def $rsp drops the tracked reload on its
+# own; only treating the call as a memory clobber keeps the second reload.
+name: call_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+ hasCalls: true
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rbp
+
+ ; CHECK-LABEL: name: call_blocks_reload_reuse
+ ; CHECK: $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: CALL64pcrel32
+ ; CHECK-NEXT: $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -24, $noreg, $rbx :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
+ CALL64pcrel32 target-flags(x86-plt) @sink, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
+ $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rbp, 1, $noreg, -24, $noreg, $rbx :: (store (s64) into %stack.1)
+ RET64
+...
+---
+name: reload_reused_from_predecessors
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ ; CHECK-LABEL: name: reload_reused_from_predecessors
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $edi
+
+ TEST32rr $edi, $edi, implicit-def $eflags
+ JCC_1 %bb.2, 5, implicit $eflags
+
+ bb.1:
+ successors: %bb.3
+
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ JMP_1 %bb.3
+
+ bb.2:
+ successors: %bb.3
+
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+
+ bb.3:
+ liveins: $rax
+
+ ; CHECK: bb.3:
+ ; CHECK-NOT: MOV64rm
+ ; CHECK: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
index 41c89355359506..588eccdae2e674 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
@@ -2291,7 +2291,6 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,4,7]
; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
; SSE-NEXT: movapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
; SSE-NEXT: movdqa %xmm10, %xmm15
; SSE-NEXT: psrld $16, %xmm15
; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
index 4abf5ccbd81ed2..616597a47d7d14 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -17625,11 +17625,10 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 116(%esp,%edx), %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 120(%esp,%edx), %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edx,%edx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
>From e5e2c90d859ddbec7eb2032e8b00b2eda42f78cf Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Tue, 1 Sep 2026 12:56:34 +0530
Subject: [PATCH 3/8] [MachineLateInstrsCleanup] Reuse reloads across more
kinds of store
A reload currently survives an intervening store only when that store is
itself a single spill-slot access naming a different slot. Everything else
drops it, which is stricter than it needs to be. A store to an LLVM IR value
cannot reach a spill slot either, so a reload could survive that too.
Replace the comparison with mayWriteSpillSlot(), which walks every memory
operand of the storing instruction and keeps the reload when none of them may
name the slot it reads. An operand carrying an IR value is skipped, and so is
one naming a different frame object, because distinct frame objects do not
overlap.
Give up whenever the memory operands do not describe everything the
instruction writes: a call, an instruction with unmodeled side effects, and an
instruction with no memory operand at all. A frame-setup push is an example of
the last, storing to the stack with nothing to describe the write.
Add tests for a store through an IR pointer and a store to an alloca, which
now keep the reload, and for a store with no pointer information and one with
no memory operand, which do not.
A last test guards the precondition the whole thing rests on. It loads from a
stack object which is not a spill slot, then stores to that same object
through its IR value. mayWriteSpillSlot() skips an operand naming an IR value,
so it would report the object untouched and the load reusable, which would
return the value from before the store. What prevents that is
isSingleSpillSlotAccess() refusing to track the load at all.
---
llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp | 53 +++++-
.../X86/machine-latecleanup-spill-reloads.mir | 155 ++++++++++++++++++
2 files changed, 200 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index 1d3a2248882637..9ab03a109e66f4 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -190,9 +190,45 @@ static bool isSingleSpillSlotAccess(const MachineInstr &MI,
return MFI.isSpillSlotObjectIndex(FI);
}
+// Return true unless MI provably cannot write the spill slot FI. The address of
+// a spill slot never escapes, so only an instruction that names the slot can
+// write it. An instruction whose memory effects are not fully described must
+// however be assumed to name it: a call may be a statepoint whose collector
+// rewrites the frame, and a frame-setup push stores with no memory operand at
+// all.
+static bool mayWriteSpillSlot(const MachineInstr &MI, int FI,
+ const MachineFrameInfo &MFI) {
+ assert(MFI.isSpillSlotObjectIndex(FI) &&
+ "Only a spill slot is known not to alias any IR value");
+ // A call or an instruction with unmodeled side effects writes more than its
+ // memory operands describe, and an empty list describes nothing at all, so
+ // neither leaves a description to prove the slot untouched with.
+ if (MI.isCall() || MI.hasUnmodeledSideEffects() || MI.memoperands_empty())
+ return true;
+
+ // Past that point the list names every location MI accesses, so it is enough
+ // to show that none of them is this slot. Whether an operand loads or stores
+ // says nothing about which location it names, and a read-modify-write may
+ // describe both of its accesses with a single operand flagged as both, so
+ // consider them all.
+ for (const MachineMemOperand *MMO : MI.memoperands()) {
+ // An access to an LLVM IR value cannot reach a spill slot.
+ if (MMO->getValue())
+ continue;
+ // Distinct frame objects do not overlap, so only an access to this very
+ // slot matters. Any other kind of pseudo value, and the absence of any
+ // pointer information, may reach the frame.
+ const auto *FSPSV =
+ dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->getPseudoValue());
+ if (!FSPSV || FSPSV->getFrameIndex() == FI)
+ return true;
+ }
+ return false;
+}
+
// Return true if MI may write the memory read by Reload, so that Reload can no
// longer be reused. The only tracked loads are invariant ones and spill-slot
-// reloads, so a store to a different spill slot cannot change the loaded value.
+// reloads.
static bool mayOverwriteReload(const MachineInstr &MI,
const MachineInstr &Reload,
const MachineFrameInfo &MFI) {
@@ -200,15 +236,16 @@ static bool mayOverwriteReload(const MachineInstr &MI,
// objects, keep reading the same value whatever memory MI writes.
if (!Reload.mayLoad() || Reload.isDereferenceableInvariantLoad())
return false;
+ // An instruction that writes no memory cannot overwrite the slot.
if (!MI.mayStore() && !MI.isCall() && !MI.hasUnmodeledSideEffects())
return false;
- int StoreFI, ReloadFI;
- if (isSingleSpillSlotAccess(MI, MFI, StoreFI) &&
- isSingleSpillSlotAccess(Reload, MFI, ReloadFI))
- return StoreFI == ReloadFI;
- // Anything else may reach the slot, e.g. a call or a store through a
- // pointer to an escaped stack object.
- return true;
+ // Reload is either invariant, and already handled above, or a spill-slot
+ // reload, so this also picks up the slot it reads. The bail-out is therefore
+ // unreachable today, and errs towards dropping the reload if that changes.
+ int ReloadFI;
+ if (!isSingleSpillSlotAccess(Reload, MFI, ReloadFI))
+ return true;
+ return mayWriteSpillSlot(MI, ReloadFI, MFI);
}
// Return true if MI is a potential candidate for reuse/removal and if so
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
index 4265c32deed012..256ffb9f6d9984 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
@@ -26,6 +26,28 @@
ret void
}
+ define void @store_through_ir_pointer_keeps_reload_reusable(ptr %p) {
+ ret void
+ }
+
+ define void @store_to_alloca_keeps_reload_reusable() {
+ %a = alloca i64
+ ret void
+ }
+
+ define void @store_without_pointer_info_blocks_reload_reuse(ptr %p) {
+ ret void
+ }
+
+ define void @store_without_memoperand_blocks_reload_reuse(ptr %p) {
+ ret void
+ }
+
+ define void @non_spill_slot_reload_is_not_reused_across_ir_store() {
+ %a = alloca i64
+ ret void
+ }
+
declare void @sink()
attributes #0 = { "frame-pointer"="all" }
@@ -194,3 +216,136 @@ body: |
MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
RET64
...
+---
+# A spill slot is created by the register allocator and its address never
+# reaches the IR, so a store that names an IR value cannot write it.
+name: store_through_ir_pointer_keeps_reload_reusable
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdi, $rdx
+
+ ; CHECK-LABEL: name: store_through_ir_pointer_keeps_reload_reusable
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %ir.p)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %ir.p)
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+# Same reasoning for an alloca: it is an IR object, so it is not the spill slot.
+name: store_to_alloca_keeps_reload_reusable
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+ - { id: 2, name: a, offset: -32, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdx
+
+ ; CHECK-LABEL: name: store_to_alloca_keeps_reload_reusable
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 48, $noreg, $rdx :: (store (s64) into %ir.a)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 48, $noreg, $rdx :: (store (s64) into %ir.a)
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+# Nothing says where this store lands, so it has to be assumed to reach the slot.
+name: store_without_pointer_info_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdi, $rdx
+
+ ; CHECK-LABEL: name: store_without_pointer_info_blocks_reload_reuse
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx :: (store (s64))
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx :: (store (s64))
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+# An empty memory operand list describes nothing, so the store may reach the slot.
+name: store_without_memoperand_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdi, $rdx
+
+ ; CHECK-LABEL: name: store_without_memoperand_blocks_reload_reuse
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+# Guards the precondition of mayWriteSpillSlot(): the load reads a frame object
+# that is not a spill slot, and the store names that very object through its IR
+# value. Reusing the load here would return the value from before the store, so
+# isSingleSpillSlotAccess() must refuse to track it in the first place.
+name: non_spill_slot_reload_is_not_reused_across_ir_store
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 48
+ maxAlignment: 8
+stack:
+ - { id: 0, name: a, offset: -16, size: 8, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -24, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdx
+
+ ; CHECK-LABEL: name: non_spill_slot_reload_is_not_reused_across_ir_store
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0.a)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 32, $noreg, $rdx :: (store (s64) into %ir.a)
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0.a)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 32, $noreg, $rdx :: (store (s64) into %ir.a)
+ $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
>From 743a9ea6da7f1020c623106051b270101dc25206 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Tue, 1 Sep 2026 13:02:15 +0530
Subject: [PATCH 4/8] [MachineLateInstrsCleanup] Add tests for a partial write
to a tracked slot
A spill slot is not always accessed at a single width. StackSlotColoring packs
several values into one slot when their live ranges do not overlap, and sizes
that slot to fit the largest of them, so one index can hold a four byte value
at one point in the function and an eight byte value at another.
mayWriteSpillSlot() compares frame indices and ignores offsets and widths, so
any write to a tracked slot ends the reuse of a reload from it, whether or not
the bytes really overlap. Add a test for each side of that: a store which
overlaps the reloaded bytes, where blocking the reuse is required, and a
disjoint one, where blocking it is merely conservative.
Reuse itself is unaffected, because hasIdentical() matches with isIdenticalTo()
and a reload of a different width is a different instruction.
Test only, no functional change.
---
.../X86/machine-latecleanup-spill-reloads.mir | 58 +++++++++++++++++++
1 file changed, 58 insertions(+)
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
index 256ffb9f6d9984..088ebdf54dca02 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
@@ -48,6 +48,14 @@
ret void
}
+ define void @same_spill_slot_partial_store_blocks_reload_reuse() {
+ ret void
+ }
+
+ define void @same_spill_slot_disjoint_store_blocks_reload_reuse() {
+ ret void
+ }
+
declare void @sink()
attributes #0 = { "frame-pointer"="all" }
@@ -349,3 +357,53 @@ body: |
MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
RET64
...
+---
+name: same_spill_slot_partial_store_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 64
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -24, size: 16, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -32, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rcx
+
+ ; CHECK-LABEL: name: same_spill_slot_partial_store_blocks_reload_reuse
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV8mr $rsp, 1, $noreg, 44, $noreg, $cl :: (store (s8) into %stack.0 + 4, align 4)
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 56, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ MOV8mr $rsp, 1, $noreg, 44, $noreg, $cl :: (store (s8) into %stack.0 + 4, align 4)
+ $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 56, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
+---
+name: same_spill_slot_disjoint_store_blocks_reload_reuse
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 64
+ maxAlignment: 8
+stack:
+ - { id: 0, type: spill-slot, offset: -24, size: 16, alignment: 8 }
+ - { id: 1, type: spill-slot, offset: -32, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rcx
+
+ ; CHECK-LABEL: name: same_spill_slot_disjoint_store_blocks_reload_reuse
+ ; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV8mr $rsp, 1, $noreg, 52, $noreg, $cl :: (store (s8) into %stack.0 + 12, align 4)
+ ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 56, $noreg, $rax :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: RET64
+ $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ MOV8mr $rsp, 1, $noreg, 52, $noreg, $cl :: (store (s8) into %stack.0 + 12, align 4)
+ $rax = MOV64rm $rsp, 1, $noreg, 40, $noreg :: (load (s64) from %stack.0)
+ MOV64mr $rsp, 1, $noreg, 56, $noreg, $rax :: (store (s64) into %stack.1)
+ RET64
+...
>From e5da08b31a28d875829d1202d9073521b79be259 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Fri, 25 Sep 2026 11:49:13 +0530
Subject: [PATCH 5/8] [AArch64] Restore x22 from its own slot in wineh1.mir
The epilogue of this hand-written test reloads x21 from %stack.2 twice. The
second one sits where the restore of x22 belongs: it is covered by a
SEH_SaveReg 22, 56 directive, the prologue saves x22 with
STRXui killed $x22, $sp, 7 :: (store (s64) into %stack.3), and the unwind
bytecode the test expects reads "ldr x22, [sp, #56]".
As written the function clobbers x22 and never restores it. Nothing noticed
because the unwind opcodes come from the SEH pseudo rather than from the load
it describes, so the directive was right and only the instruction under it was
wrong.
Load x22 from %stack.3 instead. The test output is unchanged.
---
llvm/test/CodeGen/AArch64/wineh1.mir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AArch64/wineh1.mir b/llvm/test/CodeGen/AArch64/wineh1.mir
index 244e05854e74fd..01365d00978d35 100644
--- a/llvm/test/CodeGen/AArch64/wineh1.mir
+++ b/llvm/test/CodeGen/AArch64/wineh1.mir
@@ -136,7 +136,7 @@ body: |
frame-destroy SEH_SaveReg 21, 48
$x0 = COPY $x28
frame-destroy SEH_Nop
- $x21 = frame-destroy LDRXui $sp, 6 :: (load (s64) from %stack.2)
+ $x22 = frame-destroy LDRXui $sp, 7 :: (load (s64) from %stack.3)
frame-destroy SEH_SaveReg 22, 56
$x23, $x24 = frame-destroy LDPXi $sp, 4 :: (load (s64) from %stack.4), (load (s64) from %stack.5)
frame-destroy SEH_SaveRegP 23, 24, 32
>From 03e56c82285ca21d941bfcb855a1d0a7d0416e2b Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Fri, 25 Sep 2026 11:07:28 +0530
Subject: [PATCH 6/8] [MachineLateInstrsCleanup] Reuse MachineLICM's
InstructionStoresToFI
mayWriteSpillSlot() and mayOverwriteReload() answer a question MachineLICM
already answers with InstructionStoresToFI(): does this instruction store to
this frame index. Drop both and call a copy of that helper instead, so the two
passes reason about a spill slot the same way.
The behaviour this changes is what happens to a tracked reload when an
instruction that is not a store comes in between. mayWriteSpillSlot() gave up
on a call and on anything with unmodeled side effects, on the grounds that
their memory operands do not describe everything they touch.
InstructionStoresToFI() starts from mayStore() instead, so neither ends the
reuse of a reload any more. A callee cannot store to its caller's spill slot,
and an instruction that does not store cannot write one either.
An instruction that does store keeps being handled conservatively: an empty
memory operand list still invalidates every tracked reload, which is what
covers a frame-setup push.
Also count the reloads this removes in a new statistic, and guard the entries
that are not reloads: an address computation, an immediate and an invariant
load are still dropped solely by modifiesRegister(), never by a store.
Update the tests for the call and the store with no pointer information, which
now keep the reload, and add one showing that an immediate survives a store
with no memory operand.
---
llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp | 83 +++++++------------
.../machine-latecleanup-invariant-loads.mir | 3 +-
.../X86/machine-latecleanup-spill-reloads.mir | 55 ++++++++----
3 files changed, 69 insertions(+), 72 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index 9ab03a109e66f4..f4faa3e2fda981 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -41,6 +41,8 @@ using namespace llvm;
#define DEBUG_TYPE "machine-latecleanup"
STATISTIC(NumRemoved, "Number of redundant instructions removed.");
+STATISTIC(NumSpillSlotReloadsRemoved,
+ "Number of redundant spill-slot reloads removed.");
namespace {
@@ -190,64 +192,27 @@ static bool isSingleSpillSlotAccess(const MachineInstr &MI,
return MFI.isSpillSlotObjectIndex(FI);
}
-// Return true unless MI provably cannot write the spill slot FI. The address of
-// a spill slot never escapes, so only an instruction that names the slot can
-// write it. An instruction whose memory effects are not fully described must
-// however be assumed to name it: a call may be a statepoint whose collector
-// rewrites the frame, and a frame-setup push stores with no memory operand at
-// all.
-static bool mayWriteSpillSlot(const MachineInstr &MI, int FI,
- const MachineFrameInfo &MFI) {
- assert(MFI.isSpillSlotObjectIndex(FI) &&
- "Only a spill slot is known not to alias any IR value");
- // A call or an instruction with unmodeled side effects writes more than its
- // memory operands describe, and an empty list describes nothing at all, so
- // neither leaves a description to prove the slot untouched with.
- if (MI.isCall() || MI.hasUnmodeledSideEffects() || MI.memoperands_empty())
+// Return true if MI stores to the spill slot FI. The address of a spill slot
+// never escapes, so only an instruction that describes a store to this very
+// slot can write it. An instruction that writes memory without describing it
+// at all must be assumed to write every slot. Modeled on MachineLICM's
+// InstructionStoresToFI().
+static bool instructionStoresToFI(const MachineInstr *MI, int FI) {
+ if (!MI->mayStore())
+ return false;
+ if (MI->memoperands_empty())
return true;
-
- // Past that point the list names every location MI accesses, so it is enough
- // to show that none of them is this slot. Whether an operand loads or stores
- // says nothing about which location it names, and a read-modify-write may
- // describe both of its accesses with a single operand flagged as both, so
- // consider them all.
- for (const MachineMemOperand *MMO : MI.memoperands()) {
- // An access to an LLVM IR value cannot reach a spill slot.
- if (MMO->getValue())
+ for (const MachineMemOperand *MMO : MI->memoperands()) {
+ if (!MMO->isStore() || !MMO->getPseudoValue())
continue;
- // Distinct frame objects do not overlap, so only an access to this very
- // slot matters. Any other kind of pseudo value, and the absence of any
- // pointer information, may reach the frame.
- const auto *FSPSV =
- dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->getPseudoValue());
- if (!FSPSV || FSPSV->getFrameIndex() == FI)
- return true;
+ if (const auto *PSV =
+ dyn_cast<FixedStackPseudoSourceValue>(MMO->getPseudoValue()))
+ if (PSV->getFrameIndex() == FI)
+ return true;
}
return false;
}
-// Return true if MI may write the memory read by Reload, so that Reload can no
-// longer be reused. The only tracked loads are invariant ones and spill-slot
-// reloads.
-static bool mayOverwriteReload(const MachineInstr &MI,
- const MachineInstr &Reload,
- const MachineFrameInfo &MFI) {
- // Invariant loads, such as constant pool entries and immutable frame
- // objects, keep reading the same value whatever memory MI writes.
- if (!Reload.mayLoad() || Reload.isDereferenceableInvariantLoad())
- return false;
- // An instruction that writes no memory cannot overwrite the slot.
- if (!MI.mayStore() && !MI.isCall() && !MI.hasUnmodeledSideEffects())
- return false;
- // Reload is either invariant, and already handled above, or a spill-slot
- // reload, so this also picks up the slot it reads. The bail-out is therefore
- // unreachable today, and errs towards dropping the reload if that changes.
- int ReloadFI;
- if (!isSingleSpillSlotAccess(Reload, MFI, ReloadFI))
- return true;
- return mayWriteSpillSlot(MI, ReloadFI, MFI);
-}
-
// Return true if MI is a potential candidate for reuse/removal and if so
// also the register it defines in DefedReg. A candidate is a simple
// instruction that has only one register definition and the only reg it may
@@ -328,17 +293,25 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
if (IsCandidate && MBBDefs.hasIdentical(DefedReg, &MI)) {
LLVM_DEBUG(dbgs() << "Removing redundant instruction in "
<< printMBBReference(*MBB) << ": " << MI);
+ int FI;
+ if (isSingleSpillSlotAccess(MI, MFI, FI))
+ ++NumSpillSlotReloadsRemoved;
removeRedundantDef(&MI);
Changed = true;
continue;
}
- // Clear any entries in map that MI clobbers, either by redefining the
- // register or by writing the stack slot a reload reads.
+ // Clear any entries in map that MI clobbers. A register redefinition
+ // invalidates every kind of tracked instruction. A store that may write a
+ // spill slot invalidates only a reload of that slot; address computations,
+ // immediates and invariant loads keep being dropped solely by
+ // modifiesRegister().
MBBDefs.remove_if([&](const auto &Entry) {
Register Reg = Entry.first;
+ int FI;
if (MI.modifiesRegister(Reg, TRI) ||
- mayOverwriteReload(MI, *Entry.second, MFI)) {
+ (isSingleSpillSlotAccess(*Entry.second, MFI, FI) &&
+ instructionStoresToFI(&MI, FI))) {
MBBKills.erase(Reg);
return true;
}
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
index 5c080481096d09..c2c24adb48097f 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-invariant-loads.mir
@@ -4,8 +4,7 @@
# An invariant load always returns the same value, so the pass can reuse an
# earlier one even when a store or a call comes in between. That has to keep
# working now that a spill-slot reload is tracked as well, because the stores
-# and calls which end the reuse of a reload must not end the reuse of an
-# invariant load.
+# which end the reuse of a reload must not end the reuse of an invariant load.
#
# These tests use a 32-bit target on purpose. There a constant pool entry is
# addressed absolutely, while on x86-64 the load goes through $rip. isCandidate()
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
index 088ebdf54dca02..aec688f823a33a 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
@@ -18,7 +18,7 @@
ret void
}
- define void @call_blocks_reload_reuse() #0 {
+ define void @call_keeps_reload_reusable() #0 {
ret void
}
@@ -35,7 +35,7 @@
ret void
}
- define void @store_without_pointer_info_blocks_reload_reuse(ptr %p) {
+ define void @store_without_pointer_info_keeps_reload_reusable(ptr %p) {
ret void
}
@@ -43,6 +43,10 @@
ret void
}
+ define void @store_without_memoperand_keeps_immediate_reusable(ptr %p) {
+ ret void
+ }
+
define void @non_spill_slot_reload_is_not_reused_across_ir_store() {
%a = alloca i64
ret void
@@ -157,9 +161,10 @@ body: |
...
---
# The reload target is callee-saved and the frame register is $rbp, so neither
-# the call's regmask nor its implicit-def $rsp drops the tracked reload on its
-# own; only treating the call as a memory clobber keeps the second reload.
-name: call_blocks_reload_reuse
+# the call's regmask nor its implicit-def $rsp drops the tracked reload. A call
+# is not a store to the caller's spill slot (mayStore is false), so the second
+# reload is removed.
+name: call_keeps_reload_reusable
tracksRegLiveness: true
frameInfo:
stackSize: 48
@@ -172,10 +177,9 @@ body: |
bb.0:
liveins: $rbp
- ; CHECK-LABEL: name: call_blocks_reload_reuse
+ ; CHECK-LABEL: name: call_keeps_reload_reusable
; CHECK: $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: CALL64pcrel32
- ; CHECK-NEXT: $rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: MOV64mr $rbp, 1, $noreg, -24, $noreg, $rbx :: (store (s64) into %stack.1)
; CHECK-NEXT: RET64
$rbx = MOV64rm $rbp, 1, $noreg, -16, $noreg :: (load (s64) from %stack.0)
@@ -277,8 +281,9 @@ body: |
RET64
...
---
-# Nothing says where this store lands, so it has to be assumed to reach the slot.
-name: store_without_pointer_info_blocks_reload_reuse
+# The store describes its access but names no frame index, so it is not a
+# store to the tracked slot and the reload stays reusable.
+name: store_without_pointer_info_keeps_reload_reusable
tracksRegLiveness: true
frameInfo:
stackSize: 48
@@ -290,10 +295,9 @@ body: |
bb.0:
liveins: $rdi, $rdx
- ; CHECK-LABEL: name: store_without_pointer_info_blocks_reload_reuse
+ ; CHECK-LABEL: name: store_without_pointer_info_keeps_reload_reusable
; CHECK: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx :: (store (s64))
- ; CHECK-NEXT: $rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: MOV64mr $rsp, 1, $noreg, 40, $noreg, $rax :: (store (s64) into %stack.1)
; CHECK-NEXT: RET64
$rax = MOV64rm $rsp, 1, $noreg, 32, $noreg :: (load (s64) from %stack.0)
@@ -329,10 +333,31 @@ body: |
RET64
...
---
-# Guards the precondition of mayWriteSpillSlot(): the load reads a frame object
-# that is not a spill slot, and the store names that very object through its IR
-# value. Reusing the load here would return the value from before the store, so
-# isSingleSpillSlotAccess() must refuse to track it in the first place.
+# An empty memory operand list is conservative only for a tracked spill-slot
+# reload. An immediate is invalidated solely by modifiesRegister(), so the
+# second copy is still removed.
+name: store_without_memoperand_keeps_immediate_reusable
+tracksRegLiveness: true
+frameInfo:
+ stackSize: 8
+ maxAlignment: 8
+body: |
+ bb.0:
+ liveins: $rdi, $rdx
+
+ ; CHECK-LABEL: name: store_without_memoperand_keeps_immediate_reusable
+ ; CHECK: $rcx = MOV64ri 1
+ ; CHECK-NEXT: MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx
+ ; CHECK-NEXT: RET64
+ $rcx = MOV64ri 1
+ MOV64mr $rdi, 1, $noreg, 0, $noreg, $rdx
+ $rcx = MOV64ri 1
+ RET64
+...
+---
+# The load reads a frame object that is not a spill slot, and the store names
+# that object through its IR value. Reusing the load would return the value
+# from before the store, so isSingleSpillSlotAccess() must refuse to track it.
name: non_spill_slot_reload_is_not_reused_across_ir_store
tracksRegLiveness: true
frameInfo:
>From 99dfe74a04ef9c2325df74a44cd686d84d8ff370 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Fri, 25 Sep 2026 19:47:20 +0530
Subject: [PATCH 7/8] fixup! [MachineLateInstrsCleanup] Reuse redundant
spill-slot reloads
---
llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp | 17 +++++++++--------
.../X86/machine-latecleanup-spill-reloads.mir | 2 +-
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index f4faa3e2fda981..b10793709b399e 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -177,11 +177,12 @@ void MachineLateInstrsCleanup::removeRedundantDef(MachineInstr *MI) {
++NumRemoved;
}
-// Return true if MI accesses a register-allocation spill slot through a single
-// memory operand, and if so also the frame index of that slot in FI.
-static bool isSingleSpillSlotAccess(const MachineInstr &MI,
- const MachineFrameInfo &MFI, int &FI) {
- if (!MI.hasOneMemOperand())
+// Return true if MI is a spill-slot reload with a single memory operand. If
+// that operand names a frame index, FI is set to it, and the function returns
+// true if that index is a spill slot.
+static bool isSpillSlotReload(const MachineInstr &MI,
+ const MachineFrameInfo &MFI, int &FI) {
+ if (!MI.mayLoad() || !MI.hasOneMemOperand())
return false;
const MachineMemOperand *MMO = *MI.memoperands_begin();
const auto *PSV =
@@ -231,7 +232,7 @@ static bool isCandidate(const MachineInstr *MI, Register &DefedReg,
// is left to isSafeToMove(), which admits the invariant ones and rejects the
// volatile and atomic accesses whatever SawStore says.
int FI;
- bool SawStore = !(MI->mayLoad() && isSingleSpillSlotAccess(*MI, MFI, FI));
+ bool SawStore = !isSpillSlotReload(*MI, MFI, FI);
if (!MI->isSafeToMove(SawStore) || MI->isImplicitDef() || MI->isInlineAsm())
return false;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -294,7 +295,7 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
LLVM_DEBUG(dbgs() << "Removing redundant instruction in "
<< printMBBReference(*MBB) << ": " << MI);
int FI;
- if (isSingleSpillSlotAccess(MI, MFI, FI))
+ if (isSpillSlotReload(MI, MFI, FI))
++NumSpillSlotReloadsRemoved;
removeRedundantDef(&MI);
Changed = true;
@@ -310,7 +311,7 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
Register Reg = Entry.first;
int FI;
if (MI.modifiesRegister(Reg, TRI) ||
- (isSingleSpillSlotAccess(*Entry.second, MFI, FI) &&
+ (isSpillSlotReload(*Entry.second, MFI, FI) &&
instructionStoresToFI(&MI, FI))) {
MBBKills.erase(Reg);
return true;
diff --git a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
index aec688f823a33a..d080539729af65 100644
--- a/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
+++ b/llvm/test/CodeGen/X86/machine-latecleanup-spill-reloads.mir
@@ -357,7 +357,7 @@ body: |
---
# The load reads a frame object that is not a spill slot, and the store names
# that object through its IR value. Reusing the load would return the value
-# from before the store, so isSingleSpillSlotAccess() must refuse to track it.
+# from before the store, so isSpillSlotReload() must refuse to track it.
name: non_spill_slot_reload_is_not_reused_across_ir_store
tracksRegLiveness: true
frameInfo:
>From c8365caa6463f55d34824b9475b20e2076e0b559 Mon Sep 17 00:00:00 2001
From: Anup Kumar <Anup.Kumar at amd.com>
Date: Wed, 30 Sep 2026 12:42:10 +0530
Subject: [PATCH 8/8] fixup! [MachineLateInstrsCleanup] Reuse redundant
spill-slot reloads
---
llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp | 40 +++++++++----------
1 file changed, 19 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index b10793709b399e..7372de52641e9d 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -177,27 +177,31 @@ void MachineLateInstrsCleanup::removeRedundantDef(MachineInstr *MI) {
++NumRemoved;
}
-// Return true if MI is a spill-slot reload with a single memory operand. If
-// that operand names a frame index, FI is set to it, and the function returns
-// true if that index is a spill slot.
+// Return true if MI is a spill-slot reload that does not store and has
+// exactly one memory operand. If FI is non-null and that memory operand names
+// a frame index, *FI is set to it even when the index is not a spill slot.
static bool isSpillSlotReload(const MachineInstr &MI,
- const MachineFrameInfo &MFI, int &FI) {
- if (!MI.mayLoad() || !MI.hasOneMemOperand())
+ const MachineFrameInfo &MFI, int *FI = nullptr) {
+ if (!MI.mayLoad() || MI.mayStore() || !MI.hasOneMemOperand())
return false;
+
const MachineMemOperand *MMO = *MI.memoperands_begin();
const auto *PSV =
dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->getPseudoValue());
if (!PSV)
return false;
- FI = PSV->getFrameIndex();
- return MFI.isSpillSlotObjectIndex(FI);
+ int FrameIndex = PSV->getFrameIndex();
+ if (FI)
+ *FI = FrameIndex;
+ return MFI.isSpillSlotObjectIndex(FrameIndex);
}
// Return true if MI stores to the spill slot FI. The address of a spill slot
// never escapes, so only an instruction that describes a store to this very
// slot can write it. An instruction that writes memory without describing it
-// at all must be assumed to write every slot. Modeled on MachineLICM's
-// InstructionStoresToFI().
+// at all must be assumed to write every slot.
+// TODO: This is copied from MachineLICM::InstructionStoresToFI(). Factor the
+// two copies into one helper.
static bool instructionStoresToFI(const MachineInstr *MI, int FI) {
if (!MI->mayStore())
return false;
@@ -228,11 +232,8 @@ static bool isCandidate(const MachineInstr *MI, Register &DefedReg,
// the grounds that a store may have changed what it reads. That is too blunt
// for a spill-slot reload, where only a write to the same slot matters and
// processBlock() already drops the reload as soon as it sees one. Clear
- // SawStore for those reloads to defer to that finer check; every other load
- // is left to isSafeToMove(), which admits the invariant ones and rejects the
- // volatile and atomic accesses whatever SawStore says.
- int FI;
- bool SawStore = !isSpillSlotReload(*MI, MFI, FI);
+ // SawStore for those reloads to defer to that finer check.
+ bool SawStore = !isSpillSlotReload(*MI, MFI);
if (!MI->isSafeToMove(SawStore) || MI->isImplicitDef() || MI->isInlineAsm())
return false;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -294,8 +295,7 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
if (IsCandidate && MBBDefs.hasIdentical(DefedReg, &MI)) {
LLVM_DEBUG(dbgs() << "Removing redundant instruction in "
<< printMBBReference(*MBB) << ": " << MI);
- int FI;
- if (isSpillSlotReload(MI, MFI, FI))
+ if (isSpillSlotReload(MI, MFI))
++NumSpillSlotReloadsRemoved;
removeRedundantDef(&MI);
Changed = true;
@@ -303,15 +303,13 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
}
// Clear any entries in map that MI clobbers. A register redefinition
- // invalidates every kind of tracked instruction. A store that may write a
- // spill slot invalidates only a reload of that slot; address computations,
- // immediates and invariant loads keep being dropped solely by
- // modifiesRegister().
+ // invalidates every kind of tracked instruction, and a store that writes a
+ // spill slot invalidates a reload of that slot.
MBBDefs.remove_if([&](const auto &Entry) {
Register Reg = Entry.first;
int FI;
if (MI.modifiesRegister(Reg, TRI) ||
- (isSpillSlotReload(*Entry.second, MFI, FI) &&
+ (isSpillSlotReload(*Entry.second, MFI, &FI) &&
instructionStoresToFI(&MI, FI))) {
MBBKills.erase(Reg);
return true;
More information about the llvm-commits
mailing list