[llvm] [RISCV] Add zero-store merging and load-from-store promotion (PR #192662)
Djordje Todorovic via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 06:56:34 PDT 2026
https://github.com/djtodoro created https://github.com/llvm/llvm-project/pull/192662
Improve RISCVLoadStoreOpt with:
1. Zero Store Merging
Merges adjacent stores of zero into wider stores:
- sb zero, 0(a0) + sb zero, 1(a0) → sh zero, 0(a0)
- sh zero, 0(a0) + sh zero, 2(a0) → sw zero, 0(a0)
- sw zero, 0(a0) + sw zero, 4(a0) → sd zero, 0(a0) (RV64)
2. Load-from-Store Promotion
Eliminates redundant loads by using register operations:
- sw a1, 0(a0) + lw a2, 0(a0) → sw a1, 0(a0) + mv a2, a1
- sw zero, 0(a0) + lw a1, 0(a0) → sw zero, 0(a0) + mv a1, zero
- sw a1, 0(a0) + lbu a2, 0(a0) → sw a1, 0(a0) + andi a2, a1, 255
Basically, we implemented some techniques that LLVM already perfomrs for AARCH64.
>From 909294a279fabb0f5cf414d345f59788047e659b Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Tue, 4 Nov 2025 17:22:15 +0100
Subject: [PATCH] [RISCV] Add zero-store merging and load-from-store promotion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Improve RISCVLoadStoreOpt with:
1. Zero Store Merging
Merges adjacent stores of zero into wider stores:
- sb zero, 0(a0) + sb zero, 1(a0) → sh zero, 0(a0)
- sh zero, 0(a0) + sh zero, 2(a0) → sw zero, 0(a0)
- sw zero, 0(a0) + sw zero, 4(a0) → sd zero, 0(a0) (RV64)
2. Load-from-Store Promotion
Eliminates redundant loads by using register operations:
- sw a1, 0(a0) + lw a2, 0(a0) → sw a1, 0(a0) + mv a2, a1
- sw zero, 0(a0) + lw a1, 0(a0) → sw zero, 0(a0) + mv a1, zero
- sw a1, 0(a0) + lbu a2, 0(a0) → sw a1, 0(a0) + andi a2, a1, 255
---
.../Target/RISCV/RISCVLoadStoreOptimizer.cpp | 495 +++++++++++++++++-
.../CodeGen/RISCV/load-store-opt-promotion.ll | 212 ++++++++
.../RISCV/load-store-opt-zero-merge.ll | 137 +++++
3 files changed, 826 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
create mode 100644 llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index c4ad82408c232..b4bd307439ed6 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -14,13 +14,13 @@
// Post-allocation Zilsd decomposition: Fixes invalid LD/SD instructions if
// register allocation didn't provide suitable consecutive registers.
//
-// NOTE: The AArch64LoadStoreOpt pass performs additional optimizations such as
-// merging zero store instructions, promoting loads that read directly from a
-// preceding store, and merging base register updates with load/store
-// instructions (via pre-/post-indexed addressing). These advanced
-// transformations are not yet implemented in the RISC-V pass but represent
-// potential future enhancements for further optimizing RISC-V memory
-// operations.
+// This pass also implements:
+// 1. Zero Store Merging: Merges adjacent stores of zero into wider stores
+// 2. Load-from-Store Promotion: Replaces loads that read from recently stored
+// addresses with register moves or bit extractions
+//
+// NOTE: Pre/post-indexed addressing optimizations from AArch64 are not
+// implemented as RISC-V uses a different addressing model.
//
//===----------------------------------------------------------------------===//
@@ -39,6 +39,10 @@ using namespace llvm;
#define DEBUG_TYPE "riscv-load-store-opt"
#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"
+STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
+STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
+STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
+
// The LdStLimit limits number of instructions how far we search for load/store
// pairs.
static cl::opt<unsigned> LdStLimit("riscv-load-store-scan-limit", cl::init(128),
@@ -48,6 +52,85 @@ STATISTIC(NumSD2SW, "Number of SD instructions split back to SW");
namespace {
+static bool isPromotableZeroStoreInst(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ return (Opc == RISCV::SW || Opc == RISCV::SB || Opc == RISCV::SH ||
+ Opc == RISCV::SD) &&
+ MI.getOperand(0).getReg() == RISCV::X0;
+}
+
+static bool isPromotableLoadFromStore(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ default:
+ return false;
+ case RISCV::LW:
+ case RISCV::LH:
+ case RISCV::LHU:
+ case RISCV::LB:
+ case RISCV::LBU:
+ case RISCV::LD:
+ return true;
+ }
+}
+
+static std::optional<unsigned> getMatchingWideOpcode(unsigned Opc, bool IsRV64) {
+ switch (Opc) {
+ default:
+ return std::nullopt;
+ case RISCV::SB:
+ return RISCV::SH;
+ case RISCV::SH:
+ return RISCV::SW;
+ case RISCV::SW:
+ // SW can only be widened to SD on RV64
+ if (IsRV64)
+ return RISCV::SD;
+ return std::nullopt;
+ }
+}
+
+static bool isMatchingStore(const MachineInstr &LoadInst,
+ const MachineInstr &StoreInst) {
+ unsigned LdOpc = LoadInst.getOpcode();
+ unsigned StOpc = StoreInst.getOpcode();
+ switch (LdOpc) {
+ default:
+ return false;
+ case RISCV::LB:
+ case RISCV::LBU:
+ return StOpc == RISCV::SB || StOpc == RISCV::SH || StOpc == RISCV::SW ||
+ StOpc == RISCV::SD;
+ case RISCV::LH:
+ case RISCV::LHU:
+ return StOpc == RISCV::SH || StOpc == RISCV::SW || StOpc == RISCV::SD;
+ case RISCV::LW:
+ return StOpc == RISCV::SW || StOpc == RISCV::SD;
+ case RISCV::LD:
+ return StOpc == RISCV::SD;
+ }
+}
+
+static int getMemScale(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode");
+ case RISCV::LB:
+ case RISCV::LBU:
+ case RISCV::SB:
+ return 1;
+ case RISCV::LH:
+ case RISCV::LHU:
+ case RISCV::SH:
+ return 2;
+ case RISCV::LW:
+ case RISCV::SW:
+ return 4;
+ case RISCV::LD:
+ case RISCV::SD:
+ return 8;
+ }
+}
+
struct RISCVLoadStoreOpt : public MachineFunctionPass {
static char ID;
bool runOnMachineFunction(MachineFunction &Fn) override;
@@ -68,6 +151,12 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
// Find and pair load/store instructions.
bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
+ // Find and merge zero store instructions.
+ bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
+
+ // Find and promote load instructions which read directly from store.
+ bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
+
// Convert load/store pairs to single instructions.
bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
MachineBasicBlock::iterator Second);
@@ -83,7 +172,23 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
// with the current instruction into a load/store pair.
// Return the matching instruction if one is found, else MBB->end().
MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
- bool &MergeForward);
+ bool &MergeForward,
+ bool FindNarrowMerge);
+
+ // Scan the instructions looking for a store that writes to the address from
+ // which the current load instruction reads. Return true if one is found.
+ bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
+ MachineBasicBlock::iterator &StoreI);
+
+ // Merge the two instructions indicated into a wider narrow store instruction.
+ MachineBasicBlock::iterator
+ mergeNarrowZeroStores(MachineBasicBlock::iterator I,
+ MachineBasicBlock::iterator MergeMI, bool MergeForward);
+
+ // Promote the load that reads directly from the address stored to.
+ MachineBasicBlock::iterator
+ promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
+ MachineBasicBlock::iterator StoreI);
MachineBasicBlock::iterator
mergePairedInsns(MachineBasicBlock::iterator I,
@@ -129,6 +234,21 @@ bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
MBBI != E;) {
+ // Try load-from-store promotion first
+ if (isPromotableLoadFromStore(*MBBI) &&
+ tryToPromoteLoadFromStore(MBBI)) {
+ MadeChange = true;
+ continue;
+ }
+
+ // Try zero store merging
+ if (isPromotableZeroStoreInst(*MBBI) &&
+ tryToMergeZeroStInst(MBBI)) {
+ MadeChange = true;
+ continue;
+ }
+
+ // Try load/store pairing
if (TII->isPairableLdStInstOpc(MBBI->getOpcode()) &&
tryToPairLdStInst(MBBI))
MadeChange = true;
@@ -175,8 +295,10 @@ bool RISCVLoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
// Look ahead for a pairable instruction.
MachineBasicBlock::iterator E = MI.getParent()->end();
bool MergeForward;
- MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, MergeForward);
+ MachineBasicBlock::iterator Paired =
+ findMatchingInsn(MBBI, MergeForward, /*FindNarrowMerge=*/false);
if (Paired != E) {
+ ++NumPairCreated;
MBBI = mergePairedInsns(MBBI, Paired, MergeForward);
return true;
}
@@ -479,6 +601,38 @@ bool RISCVLoadStoreOpt::tryConvertToXqcilsmLdStPair(
return true;
}
+// Merge adjacent zero stores into a wider store.
+bool RISCVLoadStoreOpt::tryToMergeZeroStInst(
+ MachineBasicBlock::iterator &MBBI) {
+ assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow zero store.");
+ MachineInstr &MI = *MBBI;
+ MachineBasicBlock::iterator E = MI.getParent()->end();
+
+ // Don't merge volatile stores
+ if (MI.hasOrderedMemoryRef())
+ return false;
+
+ // Don't optimize frame setup/destroy instructions (stack spills)
+ if (MI.getFlag(MachineInstr::FrameSetup) ||
+ MI.getFlag(MachineInstr::FrameDestroy))
+ return false;
+
+ // Skip if there is no wider store on this target (e.g. SW on RV32).
+ if (!getMatchingWideOpcode(MI.getOpcode(), STI->is64Bit()))
+ return false;
+
+ // Look ahead up to LdStLimit instructions for a mergeable instruction.
+ bool MergeForward;
+ MachineBasicBlock::iterator MergeMI =
+ findMatchingInsn(MBBI, MergeForward, /*FindNarrowMerge=*/true);
+ if (MergeMI != E) {
+ ++NumZeroStoresPromoted;
+ MBBI = mergeNarrowZeroStores(MBBI, MergeMI, MergeForward);
+ return true;
+ }
+ return false;
+}
+
bool RISCVLoadStoreOpt::tryConvertToMIPSLdStPair(
MachineFunction *MF, MachineBasicBlock::iterator First,
MachineBasicBlock::iterator Second) {
@@ -569,18 +723,23 @@ static bool mayAlias(MachineInstr &MIa,
// liveness, and potential scheduling hazards.
MachineBasicBlock::iterator
RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
- bool &MergeForward) {
+ bool &MergeForward, bool FindNarrowMerge) {
MachineBasicBlock::iterator E = I->getParent()->end();
MachineBasicBlock::iterator MBBI = I;
MachineInstr &FirstMI = *I;
MBBI = next_nodbg(MBBI, E);
+ // Only handle immediate offsets
+ if (!FirstMI.getOperand(2).isImm())
+ return E;
+
bool MayLoad = FirstMI.mayLoad();
Register Reg = FirstMI.getOperand(0).getReg();
Register BaseReg = FirstMI.getOperand(1).getReg();
int64_t Offset = FirstMI.getOperand(2).getImm();
int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();
+ bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
MergeForward = false;
// Track which register units have been modified and used between the first
@@ -600,18 +759,52 @@ RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
if (!MI.isTransient())
++Count;
- if (MI.getOpcode() == FirstMI.getOpcode() &&
- TII->isLdStSafeToPair(MI, TRI)) {
+ // For narrow merges, we need to check for matching zero stores
+ // For regular pairing, check for matching opcode and pairable instructions
+ bool IsCandidate = false;
+ if (FindNarrowMerge) {
+ // Require same opcode so that two SBs merge to SH, two SHs to SW, etc.
+ IsCandidate = IsPromotableZeroStore && isPromotableZeroStoreInst(MI) &&
+ MI.getOpcode() == FirstMI.getOpcode();
+ } else {
+ IsCandidate =
+ MI.getOpcode() == FirstMI.getOpcode() && TII->isLdStSafeToPair(MI, TRI);
+ }
+
+ if (IsCandidate) {
+ // Only handle immediate offsets
+ if (!MI.getOperand(2).isImm()) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
+ TRI);
+ if (MI.mayLoadOrStore())
+ MemInsns.push_back(&MI);
+ continue;
+ }
+
Register MIBaseReg = MI.getOperand(1).getReg();
int64_t MIOffset = MI.getOperand(2).getImm();
+ int64_t MIOffsetStride =
+ FindNarrowMerge ? getMemScale(MI) : OffsetStride;
if (BaseReg == MIBaseReg) {
- if ((Offset != MIOffset + OffsetStride) &&
- (Offset + OffsetStride != MIOffset)) {
- LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
- TRI);
- MemInsns.push_back(&MI);
- continue;
+ // For narrow merges, check if offsets are adjacent
+ if (FindNarrowMerge) {
+ if ((Offset != MIOffset + MIOffsetStride) &&
+ (Offset + OffsetStride != MIOffset)) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
+ } else {
+ // For regular pairing
+ if ((Offset != MIOffset + OffsetStride) &&
+ (Offset + OffsetStride != MIOffset)) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
}
// If the destination register of one load is the same register or a
@@ -679,6 +872,272 @@ RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
return E;
}
+// Merge two adjacent zero stores into a single wider store.
+MachineBasicBlock::iterator RISCVLoadStoreOpt::mergeNarrowZeroStores(
+ MachineBasicBlock::iterator I, MachineBasicBlock::iterator MergeMI,
+ bool MergeForward) {
+ assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
+ "Expected promotable zero stores.");
+
+ MachineBasicBlock::iterator E = I->getParent()->end();
+ MachineBasicBlock::iterator NextI = next_nodbg(I, E);
+ // If NextI is the second of the two instructions to be merged, we need
+ // to skip one further. Either way we merge will invalidate the iterator,
+ // and we don't need to scan the new instruction, as it's a pairwise
+ // instruction, which we're not considering for further action anyway.
+ if (NextI == MergeMI)
+ NextI = next_nodbg(NextI, E);
+
+ // Only handle immediate offsets
+ if (!I->getOperand(2).isImm() || !MergeMI->getOperand(2).isImm())
+ return NextI;
+
+ unsigned Opc = I->getOpcode();
+
+ // Insert our new wider store after whichever of the two stores
+ // MergeForward indicates.
+ MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
+ // Also based on MergeForward is from where we copy the base register operand
+ // so we get the flags compatible with the input code.
+ const MachineOperand &BaseRegOp =
+ MergeForward ? MergeMI->getOperand(1) : I->getOperand(1);
+
+ // RISC-V uses byte offsets; use the lower offset for the wider store.
+ int64_t IOffset = I->getOperand(2).getImm();
+ int64_t MIOffset = MergeMI->getOperand(2).getImm();
+ int64_t OffsetImm = std::min(IOffset, MIOffset);
+
+ // Get the wider opcode
+ std::optional<unsigned> NewOpcodeOpt = getMatchingWideOpcode(Opc, STI->is64Bit());
+ if (!NewOpcodeOpt)
+ return NextI; // Can't widen this instruction
+
+ unsigned NewOpcode = *NewOpcodeOpt;
+
+ // Construct the new instruction.
+ DebugLoc DL = I->getDebugLoc();
+ MachineBasicBlock *MBB = I->getParent();
+ MachineInstrBuilder MIB;
+ MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(NewOpcode))
+ .addReg(RISCV::X0)
+ .add(BaseRegOp)
+ .addImm(OffsetImm)
+ .cloneMergedMemRefs({&*I, &*MergeMI})
+ .setMIFlags(I->mergeFlagsWith(*MergeMI));
+
+ LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
+ LLVM_DEBUG(I->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " ");
+ LLVM_DEBUG(MergeMI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " with instruction:\n ");
+ LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+
+ // Erase the old instructions.
+ I->eraseFromParent();
+ MergeMI->eraseFromParent();
+ return NextI;
+}
+
+// Find a store instruction that writes to the address from which the current
+// load instruction reads. Return true if one is found.
+bool RISCVLoadStoreOpt::findMatchingStore(
+ MachineBasicBlock::iterator I, unsigned Limit,
+ MachineBasicBlock::iterator &StoreI) {
+ MachineBasicBlock::iterator B = I->getParent()->begin();
+ MachineBasicBlock::iterator MBBI = I;
+ MachineInstr &LoadMI = *I;
+
+ if (MBBI == B)
+ return false;
+
+ // Only handle immediate offset loads
+ if (!LoadMI.getOperand(2).isImm())
+ return false;
+
+ // Track which registers have been modified between the load and potential
+ // store so we can verify the store's value register is still valid.
+ ModifiedRegUnits.clear();
+ UsedRegUnits.clear();
+
+ // Look backward up to Limit instructions.
+ for (unsigned Count = 0; MBBI != B && Count < Limit;) {
+ --MBBI;
+ MachineInstr &MI = *MBBI;
+
+ // Don't count transient instructions towards the search limit.
+ if (!MI.isTransient())
+ ++Count;
+
+ if (MI.isCall())
+ return false;
+
+ if (MI.mayStore()) {
+ // Check if this is a standard store that fully covers the load range.
+ if (MI.getOperand(2).isImm() && isMatchingStore(LoadMI, MI)) {
+ Register LoadBaseReg = LoadMI.getOperand(1).getReg();
+ Register StoreBaseReg = MI.getOperand(1).getReg();
+ // RISC-V uses byte offsets; no scaling needed.
+ int64_t LoadOffset = LoadMI.getOperand(2).getImm();
+ int64_t StoreOffset = MI.getOperand(2).getImm();
+
+ int LoadSize = getMemScale(LoadMI);
+ int StoreSize = getMemScale(MI);
+
+ // Check if addresses match and load is within store range.
+ if (LoadBaseReg == StoreBaseReg && LoadOffset >= StoreOffset &&
+ (LoadOffset + LoadSize) <= (StoreOffset + StoreSize)) {
+ // Verify the store's value register hasn't been modified between
+ // the store and the load.
+ Register StRt = MI.getOperand(0).getReg();
+ if (StRt == RISCV::X0 || ModifiedRegUnits.available(StRt)) {
+ StoreI = MBBI;
+ return true;
+ }
+ // Value register was clobbered; cannot promote.
+ return false;
+ }
+ }
+
+ // This store doesn't fully cover our load. If it might modify any
+ // part of the loaded range, we must stop searching.
+ if (LoadMI.mayAlias(AA, MI, /*UseTBAA=*/false))
+ return false;
+ }
+
+ // Track register modifications as we scan backward.
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
+ }
+ return false;
+}
+
+// Promote load from store: replace a load with a register copy or bit
+// extraction when the load reads from a location that was just stored.
+MachineBasicBlock::iterator RISCVLoadStoreOpt::promoteLoadFromStore(
+ MachineBasicBlock::iterator LoadI, MachineBasicBlock::iterator StoreI) {
+ MachineBasicBlock::iterator NextI =
+ next_nodbg(LoadI, LoadI->getParent()->end());
+
+ // Both must have immediate offsets (already checked by findMatchingStore)
+ assert(LoadI->getOperand(2).isImm() && StoreI->getOperand(2).isImm() &&
+ "Expected immediate offsets");
+
+ int LoadSize = getMemScale(*LoadI);
+ int StoreSize = getMemScale(*StoreI);
+ Register LdRt = LoadI->getOperand(0).getReg();
+ Register StRt = StoreI->getOperand(0).getReg();
+
+ // Check if store is from X0 (zero register)
+ if (StRt == RISCV::X0) {
+ // Replace load with move from zero
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ADDI), LdRt)
+ .addReg(RISCV::X0)
+ .addImm(0)
+ .setMIFlags(LoadI->getFlags());
+ } else if (LoadSize == StoreSize) {
+ // If sizes match, just copy the register
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ADDI), LdRt)
+ .addReg(StRt)
+ .addImm(0)
+ .setMIFlags(LoadI->getFlags());
+ } else {
+ // For size mismatch with partial loads, we need bit extraction
+ // Since this pass runs post-RA (no virtual registers), we can only
+ // do the optimization if the load is at offset 0 of the store
+ // (no shift needed, just mask/extend)
+ // RISC-V uses byte offsets; no scaling needed.
+ int64_t LoadOffset = LoadI->getOperand(2).getImm();
+ int64_t StoreOffset = StoreI->getOperand(2).getImm();
+ int ShiftAmount = (LoadOffset - StoreOffset) * 8;
+
+ // If we need to shift, skip this optimization (would require temp register)
+ if (ShiftAmount != 0) {
+ LLVM_DEBUG(dbgs() << " Skipping promotion: would need temporary register for shift\n");
+ return NextI;
+ }
+
+ // Load is at offset 0 of the store - we can extract directly into LdRt
+ switch (LoadI->getOpcode()) {
+ case RISCV::LBU:
+ // Zero-extend byte: AND with 0xFF
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ANDI), LdRt)
+ .addReg(StRt)
+ .addImm(0xFF);
+ break;
+ case RISCV::LHU:
+ // Zero-extend halfword: use SLLI + SRLI
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI),
+ LdRt)
+ .addReg(StRt)
+ .addImm(STI->is64Bit() ? 48 : 16);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SRLI : RISCV::SRLI),
+ LdRt)
+ .addReg(LdRt)
+ .addImm(STI->is64Bit() ? 48 : 16);
+ break;
+ case RISCV::LB:
+ case RISCV::LH: {
+ // Sign-extending loads: use SLLI + SRAI
+ int ShiftBits = (STI->is64Bit() ? 64 : 32) - (LoadSize * 8);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI),
+ LdRt)
+ .addReg(StRt)
+ .addImm(ShiftBits);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SRAI : RISCV::SRAI),
+ LdRt)
+ .addReg(LdRt)
+ .addImm(ShiftBits);
+ break;
+ }
+ default:
+ // For other load types, skip optimization
+ LLVM_DEBUG(dbgs() << " Skipping promotion: unsupported load type\n");
+ return NextI;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
+ LLVM_DEBUG(StoreI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " ");
+ LLVM_DEBUG(LoadI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+
+ // Erase the old load.
+ LoadI->eraseFromParent();
+ return NextI;
+}
+
+// Try to promote a load that reads directly from a store.
+bool RISCVLoadStoreOpt::tryToPromoteLoadFromStore(
+ MachineBasicBlock::iterator &MBBI) {
+ MachineInstr &MI = *MBBI;
+
+ // If this is a volatile load, don't mess with it.
+ if (MI.hasOrderedMemoryRef())
+ return false;
+
+ // Don't optimize frame setup/destroy instructions (stack spills/reloads)
+ if (MI.getFlag(MachineInstr::FrameSetup) ||
+ MI.getFlag(MachineInstr::FrameDestroy))
+ return false;
+
+ // Look backward up to LdStLimit instructions.
+ MachineBasicBlock::iterator StoreI;
+ if (findMatchingStore(MBBI, LdStLimit, StoreI)) {
+ ++NumLoadsFromStoresPromoted;
+ MBBI = promoteLoadFromStore(MBBI, StoreI);
+ return true;
+ }
+ return false;
+}
+
MachineBasicBlock::iterator
RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
MachineBasicBlock::iterator Paired,
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll b/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
new file mode 100644
index 0000000000000..9946d80fe7795
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV64
+
+; Test load-from-store promotion: same size load/store
+define i32 @test_load_from_store_same_size(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_from_store_same_size:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_from_store_same_size:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a1
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test load-from-store promotion: load from zero store
+define i32 @test_load_from_zero_store(ptr %ptr) {
+; RV32-LABEL: test_load_from_zero_store:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: li a0, 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_from_zero_store:
+; RV64: # %bb.0:
+; RV64-NEXT: sw zero, 0(a0)
+; RV64-NEXT: li a0, 0
+; RV64-NEXT: ret
+ store i32 0, ptr %ptr, align 4
+ %val = load i32, ptr %ptr, align 4
+ ret i32 %val
+}
+
+; Test load-from-store promotion: byte load from word store (unsigned)
+define i32 @test_load_byte_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_byte_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: zext.b a2, a1
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_byte_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.b a2, a1
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_byte = load i8, ptr %ptr, align 1
+ %val_ext = zext i8 %val_byte to i32
+ ret i32 %val_ext
+}
+
+; Test load-from-store promotion: half load from word store (unsigned)
+define i32 @test_load_half_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_half_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a1, 16
+; RV32-NEXT: srli a2, a2, 16
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_half_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a1, 48
+; RV64-NEXT: srli a2, a2, 48
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_half = load i16, ptr %ptr, align 2
+ %val_ext = zext i16 %val_half to i32
+ ret i32 %val_ext
+}
+
+; Test that volatile loads are NOT promoted
+define i32 @test_volatile_load_no_promote(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_volatile_load_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: lw a0, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_volatile_load_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: lw a0, 0(a0)
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %loaded = load volatile i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test that intervening store prevents promotion
+define i32 @test_intervening_store_no_promote(ptr %ptr, i32 %val1, i32 %val2) {
+; RV32-LABEL: test_intervening_store_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_intervening_store_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val1, ptr %ptr, align 4
+ store i32 %val2, ptr %ptr, align 4
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test promotion with different base address (no promotion)
+define i32 @test_different_address_no_promote(ptr %ptr1, ptr %ptr2, i32 %val) {
+; RV32-LABEL: test_different_address_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: lw a0, 0(a1)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_different_address_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: lw a0, 0(a1)
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr1, align 4
+ %loaded = load i32, ptr %ptr2, align 4
+ ret i32 %loaded
+}
+
+; Test sign-extending byte load from word store
+define i32 @test_load_byte_sext_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_byte_sext_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a1, 24
+; RV32-NEXT: srai a2, a2, 24
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_byte_sext_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a1, 56
+; RV64-NEXT: srai a2, a2, 56
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_byte = load i8, ptr %ptr, align 1
+ %val_ext = sext i8 %val_byte to i32
+ ret i32 %val_ext
+}
+
+; Test that load after call is not promoted (call might modify memory)
+define i32 @test_call_prevents_promotion(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_call_prevents_promotion:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset ra, -4
+; RV32-NEXT: .cfi_offset s0, -8
+; RV32-NEXT: mv s0, a0
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: call bar
+; RV32-NEXT: lw a0, 0(s0)
+; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore ra
+; RV32-NEXT: .cfi_restore s0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_call_prevents_promotion:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset ra, -8
+; RV64-NEXT: .cfi_offset s0, -16
+; RV64-NEXT: mv s0, a0
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: call bar
+; RV64-NEXT: lw a0, 0(s0)
+; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore ra
+; RV64-NEXT: .cfi_restore s0
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_def_cfa_offset 0
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ call void @bar()
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+declare void @bar()
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
new file mode 100644
index 0000000000000..3d79b481d5835
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV64
+
+; Test zero store merging optimization: sb + sb -> sh
+define void @test_zero_store_merge_byte(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_byte:
+; RV32: # %bb.0:
+; RV32-NEXT: sh zero, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_byte:
+; RV64: # %bb.0:
+; RV64-NEXT: sh zero, 0(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 0, ptr %ptr1, align 1
+ ret void
+}
+
+; Test zero store merging optimization: sh + sh -> sw
+define void @test_zero_store_merge_half(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_half:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_half:
+; RV64: # %bb.0:
+; RV64-NEXT: sw zero, 0(a0)
+; RV64-NEXT: ret
+ store i16 0, ptr %ptr, align 2
+ %ptr1 = getelementptr inbounds i16, ptr %ptr, i64 1
+ store i16 0, ptr %ptr1, align 2
+ ret void
+}
+
+; Test zero store merging optimization: sw + sw -> sd (RV64 only)
+define void @test_zero_store_merge_word(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_word:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: sw zero, 4(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_word:
+; RV64: # %bb.0:
+; RV64-NEXT: sd zero, 0(a0)
+; RV64-NEXT: ret
+ store i32 0, ptr %ptr, align 4
+ %ptr1 = getelementptr inbounds i32, ptr %ptr, i64 1
+ store i32 0, ptr %ptr1, align 4
+ ret void
+}
+
+; Test that non-adjacent stores are NOT merged
+define void @test_zero_store_no_merge_non_adjacent(ptr %ptr) {
+; RV32-LABEL: test_zero_store_no_merge_non_adjacent:
+; RV32: # %bb.0:
+; RV32-NEXT: sb zero, 0(a0)
+; RV32-NEXT: sb zero, 2(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_no_merge_non_adjacent:
+; RV64: # %bb.0:
+; RV64-NEXT: sb zero, 0(a0)
+; RV64-NEXT: sb zero, 2(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr2 = getelementptr inbounds i8, ptr %ptr, i64 2
+ store i8 0, ptr %ptr2, align 1
+ ret void
+}
+
+; Test that non-zero stores are NOT merged by this optimization
+define void @test_nonzero_store_no_merge(ptr %ptr) {
+; RV32-LABEL: test_nonzero_store_no_merge:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 1
+; RV32-NEXT: sb a1, 0(a0)
+; RV32-NEXT: sb a1, 1(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_nonzero_store_no_merge:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 1
+; RV64-NEXT: sb a1, 0(a0)
+; RV64-NEXT: sb a1, 1(a0)
+; RV64-NEXT: ret
+ store i8 1, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 1, ptr %ptr1, align 1
+ ret void
+}
+
+; Test that volatile stores are NOT merged
+define void @test_volatile_zero_store_no_merge(ptr %ptr) {
+; RV32-LABEL: test_volatile_zero_store_no_merge:
+; RV32: # %bb.0:
+; RV32-NEXT: sb zero, 0(a0)
+; RV32-NEXT: sb zero, 1(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_volatile_zero_store_no_merge:
+; RV64: # %bb.0:
+; RV64-NEXT: sb zero, 0(a0)
+; RV64-NEXT: sb zero, 1(a0)
+; RV64-NEXT: ret
+ store volatile i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store volatile i8 0, ptr %ptr1, align 1
+ ret void
+}
+
+; Test merging three consecutive zero stores (should merge first two)
+define void @test_zero_store_merge_three(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_three:
+; RV32: # %bb.0:
+; RV32-NEXT: sh zero, 0(a0)
+; RV32-NEXT: sb zero, 2(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_three:
+; RV64: # %bb.0:
+; RV64-NEXT: sh zero, 0(a0)
+; RV64-NEXT: sb zero, 2(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 0, ptr %ptr1, align 1
+ %ptr2 = getelementptr inbounds i8, ptr %ptr, i64 2
+ store i8 0, ptr %ptr2, align 1
+ ret void
+}
More information about the llvm-commits
mailing list