[llvm] [RISCV] Add zero-store merging (PR #192662)
Djordje Todorovic via llvm-commits
llvm-commits at lists.llvm.org
Mon May 4 03:37:16 PDT 2026
https://github.com/djtodoro updated https://github.com/llvm/llvm-project/pull/192662
>From 559fa6043b7814895906777fb0677db3fc599b52 Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Tue, 4 Nov 2025 17:22:15 +0100
Subject: [PATCH 1/5] [RISCV] Add zero-store merging and load-from-store
promotion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Improve RISCVLoadStoreOpt with:
1. Zero Store Merging
Merges adjacent stores of zero into wider stores:
- sb zero, 0(a0) + sb zero, 1(a0) → sh zero, 0(a0)
- sh zero, 0(a0) + sh zero, 2(a0) → sw zero, 0(a0)
- sw zero, 0(a0) + sw zero, 4(a0) → sd zero, 0(a0) (RV64)
2. Load-from-Store Promotion
Eliminates redundant loads by using register operations:
- sw a1, 0(a0) + lw a2, 0(a0) → sw a1, 0(a0) + mv a2, a1
- sw zero, 0(a0) + lw a1, 0(a0) → sw zero, 0(a0) + mv a1, zero
- sw a1, 0(a0) + lbu a2, 0(a0) → sw a1, 0(a0) + andi a2, a1, 255
---
.../Target/RISCV/RISCVLoadStoreOptimizer.cpp | 495 +++++++++++++++++-
.../CodeGen/RISCV/load-store-opt-promotion.ll | 212 ++++++++
.../RISCV/load-store-opt-zero-merge.ll | 137 +++++
3 files changed, 826 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
create mode 100644 llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index c4ad82408c232..5e1f44bb41985 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -14,13 +14,13 @@
// Post-allocation Zilsd decomposition: Fixes invalid LD/SD instructions if
// register allocation didn't provide suitable consecutive registers.
//
-// NOTE: The AArch64LoadStoreOpt pass performs additional optimizations such as
-// merging zero store instructions, promoting loads that read directly from a
-// preceding store, and merging base register updates with load/store
-// instructions (via pre-/post-indexed addressing). These advanced
-// transformations are not yet implemented in the RISC-V pass but represent
-// potential future enhancements for further optimizing RISC-V memory
-// operations.
+// This pass also implements:
+// 1. Zero Store Merging: Merges adjacent stores of zero into wider stores
+// 2. Load-from-Store Promotion: Replaces loads that read from recently stored
+// addresses with register moves or bit extractions
+//
+// NOTE: Pre/post-indexed addressing optimizations from AArch64 are not
+// implemented as RISC-V uses a different addressing model.
//
//===----------------------------------------------------------------------===//
@@ -39,6 +39,10 @@ using namespace llvm;
#define DEBUG_TYPE "riscv-load-store-opt"
#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"
+STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
+STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
+STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
+
// The LdStLimit limits number of instructions how far we search for load/store
// pairs.
static cl::opt<unsigned> LdStLimit("riscv-load-store-scan-limit", cl::init(128),
@@ -48,6 +52,86 @@ STATISTIC(NumSD2SW, "Number of SD instructions split back to SW");
namespace {
+static bool isPromotableZeroStoreInst(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ return (Opc == RISCV::SW || Opc == RISCV::SB || Opc == RISCV::SH ||
+ Opc == RISCV::SD) &&
+ MI.getOperand(0).getReg() == RISCV::X0;
+}
+
+static bool isPromotableLoadFromStore(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ default:
+ return false;
+ case RISCV::LW:
+ case RISCV::LH:
+ case RISCV::LHU:
+ case RISCV::LB:
+ case RISCV::LBU:
+ case RISCV::LD:
+ return true;
+ }
+}
+
+static std::optional<unsigned> getMatchingWideOpcode(unsigned Opc,
+ bool IsRV64) {
+ switch (Opc) {
+ default:
+ return std::nullopt;
+ case RISCV::SB:
+ return RISCV::SH;
+ case RISCV::SH:
+ return RISCV::SW;
+ case RISCV::SW:
+ // SW can only be widened to SD on RV64
+ if (IsRV64)
+ return RISCV::SD;
+ return std::nullopt;
+ }
+}
+
+static bool isMatchingStore(const MachineInstr &LoadInst,
+ const MachineInstr &StoreInst) {
+ unsigned LdOpc = LoadInst.getOpcode();
+ unsigned StOpc = StoreInst.getOpcode();
+ switch (LdOpc) {
+ default:
+ return false;
+ case RISCV::LB:
+ case RISCV::LBU:
+ return StOpc == RISCV::SB || StOpc == RISCV::SH || StOpc == RISCV::SW ||
+ StOpc == RISCV::SD;
+ case RISCV::LH:
+ case RISCV::LHU:
+ return StOpc == RISCV::SH || StOpc == RISCV::SW || StOpc == RISCV::SD;
+ case RISCV::LW:
+ return StOpc == RISCV::SW || StOpc == RISCV::SD;
+ case RISCV::LD:
+ return StOpc == RISCV::SD;
+ }
+}
+
+static int getMemScale(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode");
+ case RISCV::LB:
+ case RISCV::LBU:
+ case RISCV::SB:
+ return 1;
+ case RISCV::LH:
+ case RISCV::LHU:
+ case RISCV::SH:
+ return 2;
+ case RISCV::LW:
+ case RISCV::SW:
+ return 4;
+ case RISCV::LD:
+ case RISCV::SD:
+ return 8;
+ }
+}
+
struct RISCVLoadStoreOpt : public MachineFunctionPass {
static char ID;
bool runOnMachineFunction(MachineFunction &Fn) override;
@@ -68,6 +152,12 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
// Find and pair load/store instructions.
bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
+ // Find and merge zero store instructions.
+ bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
+
+ // Find and promote load instructions which read directly from store.
+ bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
+
// Convert load/store pairs to single instructions.
bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
MachineBasicBlock::iterator Second);
@@ -83,7 +173,23 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
// with the current instruction into a load/store pair.
// Return the matching instruction if one is found, else MBB->end().
MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
- bool &MergeForward);
+ bool &MergeForward,
+ bool FindNarrowMerge);
+
+ // Scan the instructions looking for a store that writes to the address from
+ // which the current load instruction reads. Return true if one is found.
+ bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
+ MachineBasicBlock::iterator &StoreI);
+
+ // Merge the two instructions indicated into a wider narrow store instruction.
+ MachineBasicBlock::iterator
+ mergeNarrowZeroStores(MachineBasicBlock::iterator I,
+ MachineBasicBlock::iterator MergeMI, bool MergeForward);
+
+ // Promote the load that reads directly from the address stored to.
+ MachineBasicBlock::iterator
+ promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
+ MachineBasicBlock::iterator StoreI);
MachineBasicBlock::iterator
mergePairedInsns(MachineBasicBlock::iterator I,
@@ -129,6 +235,20 @@ bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
MBBI != E;) {
+ // Try load-from-store promotion first
+ if (isPromotableLoadFromStore(*MBBI) &&
+ tryToPromoteLoadFromStore(MBBI)) {
+ MadeChange = true;
+ continue;
+ }
+
+ // Try zero store merging
+ if (isPromotableZeroStoreInst(*MBBI) && tryToMergeZeroStInst(MBBI)) {
+ MadeChange = true;
+ continue;
+ }
+
+ // Try load/store pairing
if (TII->isPairableLdStInstOpc(MBBI->getOpcode()) &&
tryToPairLdStInst(MBBI))
MadeChange = true;
@@ -175,8 +295,10 @@ bool RISCVLoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
// Look ahead for a pairable instruction.
MachineBasicBlock::iterator E = MI.getParent()->end();
bool MergeForward;
- MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, MergeForward);
+ MachineBasicBlock::iterator Paired =
+ findMatchingInsn(MBBI, MergeForward, /*FindNarrowMerge=*/false);
if (Paired != E) {
+ ++NumPairCreated;
MBBI = mergePairedInsns(MBBI, Paired, MergeForward);
return true;
}
@@ -479,6 +601,38 @@ bool RISCVLoadStoreOpt::tryConvertToXqcilsmLdStPair(
return true;
}
+// Merge adjacent zero stores into a wider store.
+bool RISCVLoadStoreOpt::tryToMergeZeroStInst(
+ MachineBasicBlock::iterator &MBBI) {
+ assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow zero store.");
+ MachineInstr &MI = *MBBI;
+ MachineBasicBlock::iterator E = MI.getParent()->end();
+
+ // Don't merge volatile stores
+ if (MI.hasOrderedMemoryRef())
+ return false;
+
+ // Don't optimize frame setup/destroy instructions (stack spills)
+ if (MI.getFlag(MachineInstr::FrameSetup) ||
+ MI.getFlag(MachineInstr::FrameDestroy))
+ return false;
+
+ // Skip if there is no wider store on this target (e.g. SW on RV32).
+ if (!getMatchingWideOpcode(MI.getOpcode(), STI->is64Bit()))
+ return false;
+
+ // Look ahead up to LdStLimit instructions for a mergeable instruction.
+ bool MergeForward;
+ MachineBasicBlock::iterator MergeMI =
+ findMatchingInsn(MBBI, MergeForward, /*FindNarrowMerge=*/true);
+ if (MergeMI != E) {
+ ++NumZeroStoresPromoted;
+ MBBI = mergeNarrowZeroStores(MBBI, MergeMI, MergeForward);
+ return true;
+ }
+ return false;
+}
+
bool RISCVLoadStoreOpt::tryConvertToMIPSLdStPair(
MachineFunction *MF, MachineBasicBlock::iterator First,
MachineBasicBlock::iterator Second) {
@@ -569,18 +723,23 @@ static bool mayAlias(MachineInstr &MIa,
// liveness, and potential scheduling hazards.
MachineBasicBlock::iterator
RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
- bool &MergeForward) {
+ bool &MergeForward, bool FindNarrowMerge) {
MachineBasicBlock::iterator E = I->getParent()->end();
MachineBasicBlock::iterator MBBI = I;
MachineInstr &FirstMI = *I;
MBBI = next_nodbg(MBBI, E);
+ // Only handle immediate offsets
+ if (!FirstMI.getOperand(2).isImm())
+ return E;
+
bool MayLoad = FirstMI.mayLoad();
Register Reg = FirstMI.getOperand(0).getReg();
Register BaseReg = FirstMI.getOperand(1).getReg();
int64_t Offset = FirstMI.getOperand(2).getImm();
int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();
+ bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
MergeForward = false;
// Track which register units have been modified and used between the first
@@ -600,18 +759,51 @@ RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
if (!MI.isTransient())
++Count;
- if (MI.getOpcode() == FirstMI.getOpcode() &&
- TII->isLdStSafeToPair(MI, TRI)) {
+ // For narrow merges, we need to check for matching zero stores
+ // For regular pairing, check for matching opcode and pairable instructions
+ bool IsCandidate = false;
+ if (FindNarrowMerge) {
+ // Require same opcode so that two SBs merge to SH, two SHs to SW, etc.
+ IsCandidate = IsPromotableZeroStore && isPromotableZeroStoreInst(MI) &&
+ MI.getOpcode() == FirstMI.getOpcode();
+ } else {
+ IsCandidate = MI.getOpcode() == FirstMI.getOpcode() &&
+ TII->isLdStSafeToPair(MI, TRI);
+ }
+
+ if (IsCandidate) {
+ // Only handle immediate offsets
+ if (!MI.getOperand(2).isImm()) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
+ TRI);
+ if (MI.mayLoadOrStore())
+ MemInsns.push_back(&MI);
+ continue;
+ }
+
Register MIBaseReg = MI.getOperand(1).getReg();
int64_t MIOffset = MI.getOperand(2).getImm();
+ int64_t MIOffsetStride = FindNarrowMerge ? getMemScale(MI) : OffsetStride;
if (BaseReg == MIBaseReg) {
- if ((Offset != MIOffset + OffsetStride) &&
- (Offset + OffsetStride != MIOffset)) {
- LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
- TRI);
- MemInsns.push_back(&MI);
- continue;
+ // For narrow merges, check if offsets are adjacent
+ if (FindNarrowMerge) {
+ if ((Offset != MIOffset + MIOffsetStride) &&
+ (Offset + OffsetStride != MIOffset)) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
+ } else {
+ // For regular pairing
+ if ((Offset != MIOffset + OffsetStride) &&
+ (Offset + OffsetStride != MIOffset)) {
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
+ UsedRegUnits, TRI);
+ MemInsns.push_back(&MI);
+ continue;
+ }
}
// If the destination register of one load is the same register or a
@@ -679,6 +871,273 @@ RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
return E;
}
+// Merge two adjacent zero stores into a single wider store.
+MachineBasicBlock::iterator
+RISCVLoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
+ MachineBasicBlock::iterator MergeMI,
+ bool MergeForward) {
+ assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
+ "Expected promotable zero stores.");
+
+ MachineBasicBlock::iterator E = I->getParent()->end();
+ MachineBasicBlock::iterator NextI = next_nodbg(I, E);
+ // If NextI is the second of the two instructions to be merged, we need
+ // to skip one further. Either way we merge will invalidate the iterator,
+ // and we don't need to scan the new instruction, as it's a pairwise
+ // instruction, which we're not considering for further action anyway.
+ if (NextI == MergeMI)
+ NextI = next_nodbg(NextI, E);
+
+ // Only handle immediate offsets
+ if (!I->getOperand(2).isImm() || !MergeMI->getOperand(2).isImm())
+ return NextI;
+
+ unsigned Opc = I->getOpcode();
+
+ // Insert our new wider store after whichever of the two stores
+ // MergeForward indicates.
+ MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
+ // Also based on MergeForward is from where we copy the base register operand
+ // so we get the flags compatible with the input code.
+ const MachineOperand &BaseRegOp =
+ MergeForward ? MergeMI->getOperand(1) : I->getOperand(1);
+
+ // RISC-V uses byte offsets; use the lower offset for the wider store.
+ int64_t IOffset = I->getOperand(2).getImm();
+ int64_t MIOffset = MergeMI->getOperand(2).getImm();
+ int64_t OffsetImm = std::min(IOffset, MIOffset);
+
+ // Get the wider opcode
+ std::optional<unsigned> NewOpcodeOpt =
+ getMatchingWideOpcode(Opc, STI->is64Bit());
+ if (!NewOpcodeOpt)
+ return NextI; // Can't widen this instruction
+
+ unsigned NewOpcode = *NewOpcodeOpt;
+
+ // Construct the new instruction.
+ DebugLoc DL = I->getDebugLoc();
+ MachineBasicBlock *MBB = I->getParent();
+ MachineInstrBuilder MIB;
+ MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(NewOpcode))
+ .addReg(RISCV::X0)
+ .add(BaseRegOp)
+ .addImm(OffsetImm)
+ .cloneMergedMemRefs({&*I, &*MergeMI})
+ .setMIFlags(I->mergeFlagsWith(*MergeMI));
+
+ LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
+ LLVM_DEBUG(I->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " ");
+ LLVM_DEBUG(MergeMI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " with instruction:\n ");
+ LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+
+ // Erase the old instructions.
+ I->eraseFromParent();
+ MergeMI->eraseFromParent();
+ return NextI;
+}
+
+// Find a store instruction that writes to the address from which the current
+// load instruction reads. Return true if one is found.
+bool RISCVLoadStoreOpt::findMatchingStore(MachineBasicBlock::iterator I,
+ unsigned Limit,
+ MachineBasicBlock::iterator &StoreI) {
+ MachineBasicBlock::iterator B = I->getParent()->begin();
+ MachineBasicBlock::iterator MBBI = I;
+ MachineInstr &LoadMI = *I;
+
+ if (MBBI == B)
+ return false;
+
+ // Only handle immediate offset loads
+ if (!LoadMI.getOperand(2).isImm())
+ return false;
+
+ // Track which registers have been modified between the load and potential
+ // store so we can verify the store's value register is still valid.
+ ModifiedRegUnits.clear();
+ UsedRegUnits.clear();
+
+ // Look backward up to Limit instructions.
+ for (unsigned Count = 0; MBBI != B && Count < Limit;) {
+ --MBBI;
+ MachineInstr &MI = *MBBI;
+
+ // Don't count transient instructions towards the search limit.
+ if (!MI.isTransient())
+ ++Count;
+
+ if (MI.isCall())
+ return false;
+
+ if (MI.mayStore()) {
+ // Check if this is a standard store that fully covers the load range.
+ if (MI.getOperand(2).isImm() && isMatchingStore(LoadMI, MI)) {
+ Register LoadBaseReg = LoadMI.getOperand(1).getReg();
+ Register StoreBaseReg = MI.getOperand(1).getReg();
+ // RISC-V uses byte offsets; no scaling needed.
+ int64_t LoadOffset = LoadMI.getOperand(2).getImm();
+ int64_t StoreOffset = MI.getOperand(2).getImm();
+
+ int LoadSize = getMemScale(LoadMI);
+ int StoreSize = getMemScale(MI);
+
+ // Check if addresses match and load is within store range.
+ if (LoadBaseReg == StoreBaseReg && LoadOffset >= StoreOffset &&
+ (LoadOffset + LoadSize) <= (StoreOffset + StoreSize)) {
+ // Verify the store's value register hasn't been modified between
+ // the store and the load.
+ Register StRt = MI.getOperand(0).getReg();
+ if (StRt == RISCV::X0 || ModifiedRegUnits.available(StRt)) {
+ StoreI = MBBI;
+ return true;
+ }
+ // Value register was clobbered; cannot promote.
+ return false;
+ }
+ }
+
+ // This store doesn't fully cover our load. If it might modify any
+ // part of the loaded range, we must stop searching.
+ if (LoadMI.mayAlias(AA, MI, /*UseTBAA=*/false))
+ return false;
+ }
+
+ // Track register modifications as we scan backward.
+ LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
+ }
+ return false;
+}
+
+// Promote load from store: replace a load with a register copy or bit
+// extraction when the load reads from a location that was just stored.
+MachineBasicBlock::iterator
+RISCVLoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
+ MachineBasicBlock::iterator StoreI) {
+ MachineBasicBlock::iterator NextI =
+ next_nodbg(LoadI, LoadI->getParent()->end());
+
+ // Both must have immediate offsets (already checked by findMatchingStore)
+ assert(LoadI->getOperand(2).isImm() && StoreI->getOperand(2).isImm() &&
+ "Expected immediate offsets");
+
+ int LoadSize = getMemScale(*LoadI);
+ int StoreSize = getMemScale(*StoreI);
+ Register LdRt = LoadI->getOperand(0).getReg();
+ Register StRt = StoreI->getOperand(0).getReg();
+
+ // Check if store is from X0 (zero register)
+ if (StRt == RISCV::X0) {
+ // Replace load with move from zero
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ADDI), LdRt)
+ .addReg(RISCV::X0)
+ .addImm(0)
+ .setMIFlags(LoadI->getFlags());
+ } else if (LoadSize == StoreSize) {
+ // If sizes match, just copy the register
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ADDI), LdRt)
+ .addReg(StRt)
+ .addImm(0)
+ .setMIFlags(LoadI->getFlags());
+ } else {
+ // For size mismatch with partial loads, we need bit extraction
+ // Since this pass runs post-RA (no virtual registers), we can only
+ // do the optimization if the load is at offset 0 of the store
+ // (no shift needed, just mask/extend)
+ // RISC-V uses byte offsets; no scaling needed.
+ int64_t LoadOffset = LoadI->getOperand(2).getImm();
+ int64_t StoreOffset = StoreI->getOperand(2).getImm();
+ int ShiftAmount = (LoadOffset - StoreOffset) * 8;
+
+ // If we need to shift, skip this optimization (would require temp register)
+ if (ShiftAmount != 0) {
+ LLVM_DEBUG(
+ dbgs()
+ << " Skipping promotion: would need temporary register for shift\n");
+ return NextI;
+ }
+
+ // Load is at offset 0 of the store - we can extract directly into LdRt
+ switch (LoadI->getOpcode()) {
+ case RISCV::LBU:
+ // Zero-extend byte: AND with 0xFF
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::ANDI), LdRt)
+ .addReg(StRt)
+ .addImm(0xFF);
+ break;
+ case RISCV::LHU:
+ // Zero-extend halfword: use SLLI + SRLI
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI), LdRt)
+ .addReg(StRt)
+ .addImm(STI->is64Bit() ? 48 : 16);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SRLI : RISCV::SRLI), LdRt)
+ .addReg(LdRt)
+ .addImm(STI->is64Bit() ? 48 : 16);
+ break;
+ case RISCV::LB:
+ case RISCV::LH: {
+ // Sign-extending loads: use SLLI + SRAI
+ int ShiftBits = (STI->is64Bit() ? 64 : 32) - (LoadSize * 8);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI), LdRt)
+ .addReg(StRt)
+ .addImm(ShiftBits);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(STI->is64Bit() ? RISCV::SRAI : RISCV::SRAI), LdRt)
+ .addReg(LdRt)
+ .addImm(ShiftBits);
+ break;
+ }
+ default:
+ // For other load types, skip optimization
+ LLVM_DEBUG(dbgs() << " Skipping promotion: unsupported load type\n");
+ return NextI;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
+ LLVM_DEBUG(StoreI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << " ");
+ LLVM_DEBUG(LoadI->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+
+ // Erase the old load.
+ LoadI->eraseFromParent();
+ return NextI;
+}
+
+// Try to promote a load that reads directly from a store.
+bool RISCVLoadStoreOpt::tryToPromoteLoadFromStore(
+ MachineBasicBlock::iterator &MBBI) {
+ MachineInstr &MI = *MBBI;
+
+ // If this is a volatile load, don't mess with it.
+ if (MI.hasOrderedMemoryRef())
+ return false;
+
+ // Don't optimize frame setup/destroy instructions (stack spills/reloads)
+ if (MI.getFlag(MachineInstr::FrameSetup) ||
+ MI.getFlag(MachineInstr::FrameDestroy))
+ return false;
+
+ // Look backward up to LdStLimit instructions.
+ MachineBasicBlock::iterator StoreI;
+ if (findMatchingStore(MBBI, LdStLimit, StoreI)) {
+ ++NumLoadsFromStoresPromoted;
+ MBBI = promoteLoadFromStore(MBBI, StoreI);
+ return true;
+ }
+ return false;
+}
+
MachineBasicBlock::iterator
RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
MachineBasicBlock::iterator Paired,
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll b/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
new file mode 100644
index 0000000000000..9946d80fe7795
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV64
+
+; Test load-from-store promotion: same size load/store
+define i32 @test_load_from_store_same_size(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_from_store_same_size:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_from_store_same_size:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a1
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test load-from-store promotion: load from zero store
+define i32 @test_load_from_zero_store(ptr %ptr) {
+; RV32-LABEL: test_load_from_zero_store:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: li a0, 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_from_zero_store:
+; RV64: # %bb.0:
+; RV64-NEXT: sw zero, 0(a0)
+; RV64-NEXT: li a0, 0
+; RV64-NEXT: ret
+ store i32 0, ptr %ptr, align 4
+ %val = load i32, ptr %ptr, align 4
+ ret i32 %val
+}
+
+; Test load-from-store promotion: byte load from word store (unsigned)
+define i32 @test_load_byte_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_byte_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: zext.b a2, a1
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_byte_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.b a2, a1
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_byte = load i8, ptr %ptr, align 1
+ %val_ext = zext i8 %val_byte to i32
+ ret i32 %val_ext
+}
+
+; Test load-from-store promotion: half load from word store (unsigned)
+define i32 @test_load_half_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_half_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a1, 16
+; RV32-NEXT: srli a2, a2, 16
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_half_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a1, 48
+; RV64-NEXT: srli a2, a2, 48
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_half = load i16, ptr %ptr, align 2
+ %val_ext = zext i16 %val_half to i32
+ ret i32 %val_ext
+}
+
+; Test that volatile loads are NOT promoted
+define i32 @test_volatile_load_no_promote(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_volatile_load_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: lw a0, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_volatile_load_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: lw a0, 0(a0)
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %loaded = load volatile i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test that intervening store prevents promotion
+define i32 @test_intervening_store_no_promote(ptr %ptr, i32 %val1, i32 %val2) {
+; RV32-LABEL: test_intervening_store_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_intervening_store_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val1, ptr %ptr, align 4
+ store i32 %val2, ptr %ptr, align 4
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+; Test promotion with different base address (no promotion)
+define i32 @test_different_address_no_promote(ptr %ptr1, ptr %ptr2, i32 %val) {
+; RV32-LABEL: test_different_address_no_promote:
+; RV32: # %bb.0:
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: lw a0, 0(a1)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_different_address_no_promote:
+; RV64: # %bb.0:
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: lw a0, 0(a1)
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr1, align 4
+ %loaded = load i32, ptr %ptr2, align 4
+ ret i32 %loaded
+}
+
+; Test sign-extending byte load from word store
+define i32 @test_load_byte_sext_from_word_store(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_load_byte_sext_from_word_store:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a2, a1, 24
+; RV32-NEXT: srai a2, a2, 24
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: mv a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_load_byte_sext_from_word_store:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a2, a1, 56
+; RV64-NEXT: srai a2, a2, 56
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: mv a0, a2
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ %val_byte = load i8, ptr %ptr, align 1
+ %val_ext = sext i8 %val_byte to i32
+ ret i32 %val_ext
+}
+
+; Test that load after call is not promoted (call might modify memory)
+define i32 @test_call_prevents_promotion(ptr %ptr, i32 %val) {
+; RV32-LABEL: test_call_prevents_promotion:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset ra, -4
+; RV32-NEXT: .cfi_offset s0, -8
+; RV32-NEXT: mv s0, a0
+; RV32-NEXT: sw a1, 0(a0)
+; RV32-NEXT: call bar
+; RV32-NEXT: lw a0, 0(s0)
+; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore ra
+; RV32-NEXT: .cfi_restore s0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_call_prevents_promotion:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset ra, -8
+; RV64-NEXT: .cfi_offset s0, -16
+; RV64-NEXT: mv s0, a0
+; RV64-NEXT: sw a1, 0(a0)
+; RV64-NEXT: call bar
+; RV64-NEXT: lw a0, 0(s0)
+; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore ra
+; RV64-NEXT: .cfi_restore s0
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_def_cfa_offset 0
+; RV64-NEXT: ret
+ store i32 %val, ptr %ptr, align 4
+ call void @bar()
+ %loaded = load i32, ptr %ptr, align 4
+ ret i32 %loaded
+}
+
+declare void @bar()
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
new file mode 100644
index 0000000000000..3d79b481d5835
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN: | FileCheck %s --check-prefix=RV64
+
+; Test zero store merging optimization: sb + sb -> sh
+define void @test_zero_store_merge_byte(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_byte:
+; RV32: # %bb.0:
+; RV32-NEXT: sh zero, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_byte:
+; RV64: # %bb.0:
+; RV64-NEXT: sh zero, 0(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 0, ptr %ptr1, align 1
+ ret void
+}
+
+; Test zero store merging optimization: sh + sh -> sw
+define void @test_zero_store_merge_half(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_half:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_half:
+; RV64: # %bb.0:
+; RV64-NEXT: sw zero, 0(a0)
+; RV64-NEXT: ret
+ store i16 0, ptr %ptr, align 2
+ %ptr1 = getelementptr inbounds i16, ptr %ptr, i64 1
+ store i16 0, ptr %ptr1, align 2
+ ret void
+}
+
+; Test zero store merging optimization: sw + sw -> sd (RV64 only)
+define void @test_zero_store_merge_word(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_word:
+; RV32: # %bb.0:
+; RV32-NEXT: sw zero, 0(a0)
+; RV32-NEXT: sw zero, 4(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_word:
+; RV64: # %bb.0:
+; RV64-NEXT: sd zero, 0(a0)
+; RV64-NEXT: ret
+ store i32 0, ptr %ptr, align 4
+ %ptr1 = getelementptr inbounds i32, ptr %ptr, i64 1
+ store i32 0, ptr %ptr1, align 4
+ ret void
+}
+
+; Test that non-adjacent stores are NOT merged
+define void @test_zero_store_no_merge_non_adjacent(ptr %ptr) {
+; RV32-LABEL: test_zero_store_no_merge_non_adjacent:
+; RV32: # %bb.0:
+; RV32-NEXT: sb zero, 0(a0)
+; RV32-NEXT: sb zero, 2(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_no_merge_non_adjacent:
+; RV64: # %bb.0:
+; RV64-NEXT: sb zero, 0(a0)
+; RV64-NEXT: sb zero, 2(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr2 = getelementptr inbounds i8, ptr %ptr, i64 2
+ store i8 0, ptr %ptr2, align 1
+ ret void
+}
+
+; Test that non-zero stores are NOT merged by this optimization
+define void @test_nonzero_store_no_merge(ptr %ptr) {
+; RV32-LABEL: test_nonzero_store_no_merge:
+; RV32: # %bb.0:
+; RV32-NEXT: li a1, 1
+; RV32-NEXT: sb a1, 0(a0)
+; RV32-NEXT: sb a1, 1(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_nonzero_store_no_merge:
+; RV64: # %bb.0:
+; RV64-NEXT: li a1, 1
+; RV64-NEXT: sb a1, 0(a0)
+; RV64-NEXT: sb a1, 1(a0)
+; RV64-NEXT: ret
+ store i8 1, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 1, ptr %ptr1, align 1
+ ret void
+}
+
+; Test that volatile stores are NOT merged
+define void @test_volatile_zero_store_no_merge(ptr %ptr) {
+; RV32-LABEL: test_volatile_zero_store_no_merge:
+; RV32: # %bb.0:
+; RV32-NEXT: sb zero, 0(a0)
+; RV32-NEXT: sb zero, 1(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_volatile_zero_store_no_merge:
+; RV64: # %bb.0:
+; RV64-NEXT: sb zero, 0(a0)
+; RV64-NEXT: sb zero, 1(a0)
+; RV64-NEXT: ret
+ store volatile i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store volatile i8 0, ptr %ptr1, align 1
+ ret void
+}
+
+; Test merging three consecutive zero stores (should merge first two)
+define void @test_zero_store_merge_three(ptr %ptr) {
+; RV32-LABEL: test_zero_store_merge_three:
+; RV32: # %bb.0:
+; RV32-NEXT: sh zero, 0(a0)
+; RV32-NEXT: sb zero, 2(a0)
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_zero_store_merge_three:
+; RV64: # %bb.0:
+; RV64-NEXT: sh zero, 0(a0)
+; RV64-NEXT: sb zero, 2(a0)
+; RV64-NEXT: ret
+ store i8 0, ptr %ptr, align 1
+ %ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
+ store i8 0, ptr %ptr1, align 1
+ %ptr2 = getelementptr inbounds i8, ptr %ptr, i64 2
+ store i8 0, ptr %ptr2, align 1
+ ret void
+}
>From d167e3cfb39b496e61b907fa1d1c6603d43dc34a Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Wed, 29 Apr 2026 09:13:25 +0200
Subject: [PATCH 2/5] fix: potentially an unaligned access
---
.../Target/RISCV/RISCVLoadStoreOptimizer.cpp | 8 +++++++
.../RISCV/load-store-opt-zero-merge.ll | 21 +++++++++----------
2 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index 5e1f44bb41985..6fdd4b301430c 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -626,6 +626,14 @@ bool RISCVLoadStoreOpt::tryToMergeZeroStInst(
MachineBasicBlock::iterator MergeMI =
findMatchingInsn(MBBI, MergeForward, /*FindNarrowMerge=*/true);
if (MergeMI != E) {
+ MachineInstr &LowerMI =
+ MI.getOperand(2).getImm() <= MergeMI->getOperand(2).getImm() ? MI
+ : *MergeMI;
+ Align RequiredAlign(getMemScale(MI) * 2);
+ if (!STI->enableUnalignedScalarMem() &&
+ (*LowerMI.memoperands_begin())->getAlign() < RequiredAlign)
+ return false;
+
++NumZeroStoresPromoted;
MBBI = mergeNarrowZeroStores(MBBI, MergeMI, MergeForward);
return true;
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
index 3d79b481d5835..638b136a56e31 100644
--- a/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
+++ b/llvm/test/CodeGen/RISCV/load-store-opt-zero-merge.ll
@@ -5,7 +5,7 @@
; RUN: | FileCheck %s --check-prefix=RV64
; Test zero store merging optimization: sb + sb -> sh
-define void @test_zero_store_merge_byte(ptr %ptr) {
+define void @test_zero_store_merge_byte(ptr align 2 %ptr) {
; RV32-LABEL: test_zero_store_merge_byte:
; RV32: # %bb.0:
; RV32-NEXT: sh zero, 0(a0)
@@ -15,14 +15,14 @@ define void @test_zero_store_merge_byte(ptr %ptr) {
; RV64: # %bb.0:
; RV64-NEXT: sh zero, 0(a0)
; RV64-NEXT: ret
- store i8 0, ptr %ptr, align 1
+ store i8 0, ptr %ptr, align 2
%ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
store i8 0, ptr %ptr1, align 1
ret void
}
; Test zero store merging optimization: sh + sh -> sw
-define void @test_zero_store_merge_half(ptr %ptr) {
+define void @test_zero_store_merge_half(ptr align 4 %ptr) {
; RV32-LABEL: test_zero_store_merge_half:
; RV32: # %bb.0:
; RV32-NEXT: sw zero, 0(a0)
@@ -32,25 +32,24 @@ define void @test_zero_store_merge_half(ptr %ptr) {
; RV64: # %bb.0:
; RV64-NEXT: sw zero, 0(a0)
; RV64-NEXT: ret
- store i16 0, ptr %ptr, align 2
+ store i16 0, ptr %ptr, align 4
%ptr1 = getelementptr inbounds i16, ptr %ptr, i64 1
store i16 0, ptr %ptr1, align 2
ret void
}
; Test zero store merging optimization: sw + sw -> sd (RV64 only)
-define void @test_zero_store_merge_word(ptr %ptr) {
+define void @test_zero_store_merge_word(ptr align 8 %ptr) {
; RV32-LABEL: test_zero_store_merge_word:
; RV32: # %bb.0:
-; RV32-NEXT: sw zero, 0(a0)
-; RV32-NEXT: sw zero, 4(a0)
+; RV32-NEXT: mips.swp zero, zero, 0(a0)
; RV32-NEXT: ret
;
; RV64-LABEL: test_zero_store_merge_word:
; RV64: # %bb.0:
; RV64-NEXT: sd zero, 0(a0)
; RV64-NEXT: ret
- store i32 0, ptr %ptr, align 4
+ store i32 0, ptr %ptr, align 8
%ptr1 = getelementptr inbounds i32, ptr %ptr, i64 1
store i32 0, ptr %ptr1, align 4
ret void
@@ -116,7 +115,7 @@ define void @test_volatile_zero_store_no_merge(ptr %ptr) {
}
; Test merging three consecutive zero stores (should merge first two)
-define void @test_zero_store_merge_three(ptr %ptr) {
+define void @test_zero_store_merge_three(ptr align 2 %ptr) {
; RV32-LABEL: test_zero_store_merge_three:
; RV32: # %bb.0:
; RV32-NEXT: sh zero, 0(a0)
@@ -128,10 +127,10 @@ define void @test_zero_store_merge_three(ptr %ptr) {
; RV64-NEXT: sh zero, 0(a0)
; RV64-NEXT: sb zero, 2(a0)
; RV64-NEXT: ret
- store i8 0, ptr %ptr, align 1
+ store i8 0, ptr %ptr, align 2
%ptr1 = getelementptr inbounds i8, ptr %ptr, i64 1
store i8 0, ptr %ptr1, align 1
%ptr2 = getelementptr inbounds i8, ptr %ptr, i64 2
- store i8 0, ptr %ptr2, align 1
+ store i8 0, ptr %ptr2, align 2
ret void
}
>From e8bbb4bea5e0a7bf51a5d7dcd69b1296575a6787 Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Thu, 30 Apr 2026 16:08:32 +0200
Subject: [PATCH 3/5] [RISCV] Add LWU support to load-store optimizer
---
llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index 6fdd4b301430c..230b5d3d758d8 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -64,6 +64,7 @@ static bool isPromotableLoadFromStore(const MachineInstr &MI) {
default:
return false;
case RISCV::LW:
+ case RISCV::LWU:
case RISCV::LH:
case RISCV::LHU:
case RISCV::LB:
@@ -105,6 +106,7 @@ static bool isMatchingStore(const MachineInstr &LoadInst,
case RISCV::LHU:
return StOpc == RISCV::SH || StOpc == RISCV::SW || StOpc == RISCV::SD;
case RISCV::LW:
+ case RISCV::LWU:
return StOpc == RISCV::SW || StOpc == RISCV::SD;
case RISCV::LD:
return StOpc == RISCV::SD;
@@ -124,6 +126,7 @@ static int getMemScale(const MachineInstr &MI) {
case RISCV::SH:
return 2;
case RISCV::LW:
+ case RISCV::LWU:
case RISCV::SW:
return 4;
case RISCV::LD:
@@ -1045,6 +1048,16 @@ RISCVLoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
.addReg(RISCV::X0)
.addImm(0)
.setMIFlags(LoadI->getFlags());
+ } else if (LoadI->getOpcode() == RISCV::LWU &&
+ LoadI->getOperand(2).getImm() == StoreI->getOperand(2).getImm()) {
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::SLLI), LdRt)
+ .addReg(StRt)
+ .addImm(32);
+ BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
+ TII->get(RISCV::SRLI), LdRt)
+ .addReg(LdRt)
+ .addImm(32);
} else if (LoadSize == StoreSize) {
// If sizes match, just copy the register
BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
>From 33a290fb7d2b3baea362f406634a83eb4a4b2614 Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Thu, 30 Apr 2026 17:01:28 +0200
Subject: [PATCH 4/5] [RISCV] Drop load-from-store promotion
Keep zero-store merging only, I will post a follow-up PR.
---
.../Target/RISCV/RISCVLoadStoreOptimizer.cpp | 277 +-----------------
.../CodeGen/RISCV/load-store-opt-promotion.ll | 212 --------------
2 files changed, 1 insertion(+), 488 deletions(-)
delete mode 100644 llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index 230b5d3d758d8..d20cf4f38ebab 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -15,9 +15,7 @@
// register allocation didn't provide suitable consecutive registers.
//
// This pass also implements:
-// 1. Zero Store Merging: Merges adjacent stores of zero into wider stores
-// 2. Load-from-Store Promotion: Replaces loads that read from recently stored
-// addresses with register moves or bit extractions
+// - Zero Store Merging: Merges adjacent stores of zero into wider stores
//
// NOTE: Pre/post-indexed addressing optimizations from AArch64 are not
// implemented as RISC-V uses a different addressing model.
@@ -41,7 +39,6 @@ using namespace llvm;
STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
-STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
// The LdStLimit limits number of instructions how far we search for load/store
// pairs.
@@ -59,21 +56,6 @@ static bool isPromotableZeroStoreInst(const MachineInstr &MI) {
MI.getOperand(0).getReg() == RISCV::X0;
}
-static bool isPromotableLoadFromStore(const MachineInstr &MI) {
- switch (MI.getOpcode()) {
- default:
- return false;
- case RISCV::LW:
- case RISCV::LWU:
- case RISCV::LH:
- case RISCV::LHU:
- case RISCV::LB:
- case RISCV::LBU:
- case RISCV::LD:
- return true;
- }
-}
-
static std::optional<unsigned> getMatchingWideOpcode(unsigned Opc,
bool IsRV64) {
switch (Opc) {
@@ -91,45 +73,16 @@ static std::optional<unsigned> getMatchingWideOpcode(unsigned Opc,
}
}
-static bool isMatchingStore(const MachineInstr &LoadInst,
- const MachineInstr &StoreInst) {
- unsigned LdOpc = LoadInst.getOpcode();
- unsigned StOpc = StoreInst.getOpcode();
- switch (LdOpc) {
- default:
- return false;
- case RISCV::LB:
- case RISCV::LBU:
- return StOpc == RISCV::SB || StOpc == RISCV::SH || StOpc == RISCV::SW ||
- StOpc == RISCV::SD;
- case RISCV::LH:
- case RISCV::LHU:
- return StOpc == RISCV::SH || StOpc == RISCV::SW || StOpc == RISCV::SD;
- case RISCV::LW:
- case RISCV::LWU:
- return StOpc == RISCV::SW || StOpc == RISCV::SD;
- case RISCV::LD:
- return StOpc == RISCV::SD;
- }
-}
-
static int getMemScale(const MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected opcode");
- case RISCV::LB:
- case RISCV::LBU:
case RISCV::SB:
return 1;
- case RISCV::LH:
- case RISCV::LHU:
case RISCV::SH:
return 2;
- case RISCV::LW:
- case RISCV::LWU:
case RISCV::SW:
return 4;
- case RISCV::LD:
case RISCV::SD:
return 8;
}
@@ -158,9 +111,6 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
// Find and merge zero store instructions.
bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
- // Find and promote load instructions which read directly from store.
- bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
-
// Convert load/store pairs to single instructions.
bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
MachineBasicBlock::iterator Second);
@@ -179,21 +129,11 @@ struct RISCVLoadStoreOpt : public MachineFunctionPass {
bool &MergeForward,
bool FindNarrowMerge);
- // Scan the instructions looking for a store that writes to the address from
- // which the current load instruction reads. Return true if one is found.
- bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
- MachineBasicBlock::iterator &StoreI);
-
// Merge the two instructions indicated into a wider narrow store instruction.
MachineBasicBlock::iterator
mergeNarrowZeroStores(MachineBasicBlock::iterator I,
MachineBasicBlock::iterator MergeMI, bool MergeForward);
- // Promote the load that reads directly from the address stored to.
- MachineBasicBlock::iterator
- promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
- MachineBasicBlock::iterator StoreI);
-
MachineBasicBlock::iterator
mergePairedInsns(MachineBasicBlock::iterator I,
MachineBasicBlock::iterator Paired, bool MergeForward);
@@ -238,13 +178,6 @@ bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
MBBI != E;) {
- // Try load-from-store promotion first
- if (isPromotableLoadFromStore(*MBBI) &&
- tryToPromoteLoadFromStore(MBBI)) {
- MadeChange = true;
- continue;
- }
-
// Try zero store merging
if (isPromotableZeroStoreInst(*MBBI) && tryToMergeZeroStInst(MBBI)) {
MadeChange = true;
@@ -951,214 +884,6 @@ RISCVLoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
return NextI;
}
-// Find a store instruction that writes to the address from which the current
-// load instruction reads. Return true if one is found.
-bool RISCVLoadStoreOpt::findMatchingStore(MachineBasicBlock::iterator I,
- unsigned Limit,
- MachineBasicBlock::iterator &StoreI) {
- MachineBasicBlock::iterator B = I->getParent()->begin();
- MachineBasicBlock::iterator MBBI = I;
- MachineInstr &LoadMI = *I;
-
- if (MBBI == B)
- return false;
-
- // Only handle immediate offset loads
- if (!LoadMI.getOperand(2).isImm())
- return false;
-
- // Track which registers have been modified between the load and potential
- // store so we can verify the store's value register is still valid.
- ModifiedRegUnits.clear();
- UsedRegUnits.clear();
-
- // Look backward up to Limit instructions.
- for (unsigned Count = 0; MBBI != B && Count < Limit;) {
- --MBBI;
- MachineInstr &MI = *MBBI;
-
- // Don't count transient instructions towards the search limit.
- if (!MI.isTransient())
- ++Count;
-
- if (MI.isCall())
- return false;
-
- if (MI.mayStore()) {
- // Check if this is a standard store that fully covers the load range.
- if (MI.getOperand(2).isImm() && isMatchingStore(LoadMI, MI)) {
- Register LoadBaseReg = LoadMI.getOperand(1).getReg();
- Register StoreBaseReg = MI.getOperand(1).getReg();
- // RISC-V uses byte offsets; no scaling needed.
- int64_t LoadOffset = LoadMI.getOperand(2).getImm();
- int64_t StoreOffset = MI.getOperand(2).getImm();
-
- int LoadSize = getMemScale(LoadMI);
- int StoreSize = getMemScale(MI);
-
- // Check if addresses match and load is within store range.
- if (LoadBaseReg == StoreBaseReg && LoadOffset >= StoreOffset &&
- (LoadOffset + LoadSize) <= (StoreOffset + StoreSize)) {
- // Verify the store's value register hasn't been modified between
- // the store and the load.
- Register StRt = MI.getOperand(0).getReg();
- if (StRt == RISCV::X0 || ModifiedRegUnits.available(StRt)) {
- StoreI = MBBI;
- return true;
- }
- // Value register was clobbered; cannot promote.
- return false;
- }
- }
-
- // This store doesn't fully cover our load. If it might modify any
- // part of the loaded range, we must stop searching.
- if (LoadMI.mayAlias(AA, MI, /*UseTBAA=*/false))
- return false;
- }
-
- // Track register modifications as we scan backward.
- LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
- }
- return false;
-}
-
-// Promote load from store: replace a load with a register copy or bit
-// extraction when the load reads from a location that was just stored.
-MachineBasicBlock::iterator
-RISCVLoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
- MachineBasicBlock::iterator StoreI) {
- MachineBasicBlock::iterator NextI =
- next_nodbg(LoadI, LoadI->getParent()->end());
-
- // Both must have immediate offsets (already checked by findMatchingStore)
- assert(LoadI->getOperand(2).isImm() && StoreI->getOperand(2).isImm() &&
- "Expected immediate offsets");
-
- int LoadSize = getMemScale(*LoadI);
- int StoreSize = getMemScale(*StoreI);
- Register LdRt = LoadI->getOperand(0).getReg();
- Register StRt = StoreI->getOperand(0).getReg();
-
- // Check if store is from X0 (zero register)
- if (StRt == RISCV::X0) {
- // Replace load with move from zero
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(RISCV::ADDI), LdRt)
- .addReg(RISCV::X0)
- .addImm(0)
- .setMIFlags(LoadI->getFlags());
- } else if (LoadI->getOpcode() == RISCV::LWU &&
- LoadI->getOperand(2).getImm() == StoreI->getOperand(2).getImm()) {
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(RISCV::SLLI), LdRt)
- .addReg(StRt)
- .addImm(32);
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(RISCV::SRLI), LdRt)
- .addReg(LdRt)
- .addImm(32);
- } else if (LoadSize == StoreSize) {
- // If sizes match, just copy the register
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(RISCV::ADDI), LdRt)
- .addReg(StRt)
- .addImm(0)
- .setMIFlags(LoadI->getFlags());
- } else {
- // For size mismatch with partial loads, we need bit extraction
- // Since this pass runs post-RA (no virtual registers), we can only
- // do the optimization if the load is at offset 0 of the store
- // (no shift needed, just mask/extend)
- // RISC-V uses byte offsets; no scaling needed.
- int64_t LoadOffset = LoadI->getOperand(2).getImm();
- int64_t StoreOffset = StoreI->getOperand(2).getImm();
- int ShiftAmount = (LoadOffset - StoreOffset) * 8;
-
- // If we need to shift, skip this optimization (would require temp register)
- if (ShiftAmount != 0) {
- LLVM_DEBUG(
- dbgs()
- << " Skipping promotion: would need temporary register for shift\n");
- return NextI;
- }
-
- // Load is at offset 0 of the store - we can extract directly into LdRt
- switch (LoadI->getOpcode()) {
- case RISCV::LBU:
- // Zero-extend byte: AND with 0xFF
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(RISCV::ANDI), LdRt)
- .addReg(StRt)
- .addImm(0xFF);
- break;
- case RISCV::LHU:
- // Zero-extend halfword: use SLLI + SRLI
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI), LdRt)
- .addReg(StRt)
- .addImm(STI->is64Bit() ? 48 : 16);
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(STI->is64Bit() ? RISCV::SRLI : RISCV::SRLI), LdRt)
- .addReg(LdRt)
- .addImm(STI->is64Bit() ? 48 : 16);
- break;
- case RISCV::LB:
- case RISCV::LH: {
- // Sign-extending loads: use SLLI + SRAI
- int ShiftBits = (STI->is64Bit() ? 64 : 32) - (LoadSize * 8);
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(STI->is64Bit() ? RISCV::SLLI : RISCV::SLLI), LdRt)
- .addReg(StRt)
- .addImm(ShiftBits);
- BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
- TII->get(STI->is64Bit() ? RISCV::SRAI : RISCV::SRAI), LdRt)
- .addReg(LdRt)
- .addImm(ShiftBits);
- break;
- }
- default:
- // For other load types, skip optimization
- LLVM_DEBUG(dbgs() << " Skipping promotion: unsupported load type\n");
- return NextI;
- }
- }
-
- LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
- LLVM_DEBUG(StoreI->print(dbgs()));
- LLVM_DEBUG(dbgs() << " ");
- LLVM_DEBUG(LoadI->print(dbgs()));
- LLVM_DEBUG(dbgs() << "\n");
-
- // Erase the old load.
- LoadI->eraseFromParent();
- return NextI;
-}
-
-// Try to promote a load that reads directly from a store.
-bool RISCVLoadStoreOpt::tryToPromoteLoadFromStore(
- MachineBasicBlock::iterator &MBBI) {
- MachineInstr &MI = *MBBI;
-
- // If this is a volatile load, don't mess with it.
- if (MI.hasOrderedMemoryRef())
- return false;
-
- // Don't optimize frame setup/destroy instructions (stack spills/reloads)
- if (MI.getFlag(MachineInstr::FrameSetup) ||
- MI.getFlag(MachineInstr::FrameDestroy))
- return false;
-
- // Look backward up to LdStLimit instructions.
- MachineBasicBlock::iterator StoreI;
- if (findMatchingStore(MBBI, LdStLimit, StoreI)) {
- ++NumLoadsFromStoresPromoted;
- MBBI = promoteLoadFromStore(MBBI, StoreI);
- return true;
- }
- return false;
-}
-
MachineBasicBlock::iterator
RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
MachineBasicBlock::iterator Paired,
diff --git a/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll b/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
deleted file mode 100644
index 9946d80fe7795..0000000000000
--- a/llvm/test/CodeGen/RISCV/load-store-opt-promotion.ll
+++ /dev/null
@@ -1,212 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
-; RUN: | FileCheck %s --check-prefix=RV32
-; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
-; RUN: | FileCheck %s --check-prefix=RV64
-
-; Test load-from-store promotion: same size load/store
-define i32 @test_load_from_store_same_size(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_load_from_store_same_size:
-; RV32: # %bb.0:
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: mv a0, a1
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_load_from_store_same_size:
-; RV64: # %bb.0:
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: mv a0, a1
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- %loaded = load i32, ptr %ptr, align 4
- ret i32 %loaded
-}
-
-; Test load-from-store promotion: load from zero store
-define i32 @test_load_from_zero_store(ptr %ptr) {
-; RV32-LABEL: test_load_from_zero_store:
-; RV32: # %bb.0:
-; RV32-NEXT: sw zero, 0(a0)
-; RV32-NEXT: li a0, 0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_load_from_zero_store:
-; RV64: # %bb.0:
-; RV64-NEXT: sw zero, 0(a0)
-; RV64-NEXT: li a0, 0
-; RV64-NEXT: ret
- store i32 0, ptr %ptr, align 4
- %val = load i32, ptr %ptr, align 4
- ret i32 %val
-}
-
-; Test load-from-store promotion: byte load from word store (unsigned)
-define i32 @test_load_byte_from_word_store(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_load_byte_from_word_store:
-; RV32: # %bb.0:
-; RV32-NEXT: zext.b a2, a1
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: mv a0, a2
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_load_byte_from_word_store:
-; RV64: # %bb.0:
-; RV64-NEXT: zext.b a2, a1
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: mv a0, a2
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- %val_byte = load i8, ptr %ptr, align 1
- %val_ext = zext i8 %val_byte to i32
- ret i32 %val_ext
-}
-
-; Test load-from-store promotion: half load from word store (unsigned)
-define i32 @test_load_half_from_word_store(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_load_half_from_word_store:
-; RV32: # %bb.0:
-; RV32-NEXT: slli a2, a1, 16
-; RV32-NEXT: srli a2, a2, 16
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: mv a0, a2
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_load_half_from_word_store:
-; RV64: # %bb.0:
-; RV64-NEXT: slli a2, a1, 48
-; RV64-NEXT: srli a2, a2, 48
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: mv a0, a2
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- %val_half = load i16, ptr %ptr, align 2
- %val_ext = zext i16 %val_half to i32
- ret i32 %val_ext
-}
-
-; Test that volatile loads are NOT promoted
-define i32 @test_volatile_load_no_promote(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_volatile_load_no_promote:
-; RV32: # %bb.0:
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: lw a0, 0(a0)
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_volatile_load_no_promote:
-; RV64: # %bb.0:
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: lw a0, 0(a0)
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- %loaded = load volatile i32, ptr %ptr, align 4
- ret i32 %loaded
-}
-
-; Test that intervening store prevents promotion
-define i32 @test_intervening_store_no_promote(ptr %ptr, i32 %val1, i32 %val2) {
-; RV32-LABEL: test_intervening_store_no_promote:
-; RV32: # %bb.0:
-; RV32-NEXT: sw a2, 0(a0)
-; RV32-NEXT: mv a0, a2
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_intervening_store_no_promote:
-; RV64: # %bb.0:
-; RV64-NEXT: sw a2, 0(a0)
-; RV64-NEXT: mv a0, a2
-; RV64-NEXT: ret
- store i32 %val1, ptr %ptr, align 4
- store i32 %val2, ptr %ptr, align 4
- %loaded = load i32, ptr %ptr, align 4
- ret i32 %loaded
-}
-
-; Test promotion with different base address (no promotion)
-define i32 @test_different_address_no_promote(ptr %ptr1, ptr %ptr2, i32 %val) {
-; RV32-LABEL: test_different_address_no_promote:
-; RV32: # %bb.0:
-; RV32-NEXT: sw a2, 0(a0)
-; RV32-NEXT: lw a0, 0(a1)
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_different_address_no_promote:
-; RV64: # %bb.0:
-; RV64-NEXT: sw a2, 0(a0)
-; RV64-NEXT: lw a0, 0(a1)
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr1, align 4
- %loaded = load i32, ptr %ptr2, align 4
- ret i32 %loaded
-}
-
-; Test sign-extending byte load from word store
-define i32 @test_load_byte_sext_from_word_store(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_load_byte_sext_from_word_store:
-; RV32: # %bb.0:
-; RV32-NEXT: slli a2, a1, 24
-; RV32-NEXT: srai a2, a2, 24
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: mv a0, a2
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_load_byte_sext_from_word_store:
-; RV64: # %bb.0:
-; RV64-NEXT: slli a2, a1, 56
-; RV64-NEXT: srai a2, a2, 56
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: mv a0, a2
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- %val_byte = load i8, ptr %ptr, align 1
- %val_ext = sext i8 %val_byte to i32
- ret i32 %val_ext
-}
-
-; Test that load after call is not promoted (call might modify memory)
-define i32 @test_call_prevents_promotion(ptr %ptr, i32 %val) {
-; RV32-LABEL: test_call_prevents_promotion:
-; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: .cfi_offset s0, -8
-; RV32-NEXT: mv s0, a0
-; RV32-NEXT: sw a1, 0(a0)
-; RV32-NEXT: call bar
-; RV32-NEXT: lw a0, 0(s0)
-; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT: .cfi_restore ra
-; RV32-NEXT: .cfi_restore s0
-; RV32-NEXT: addi sp, sp, 16
-; RV32-NEXT: .cfi_def_cfa_offset 0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: test_call_prevents_promotion:
-; RV64: # %bb.0:
-; RV64-NEXT: addi sp, sp, -16
-; RV64-NEXT: .cfi_def_cfa_offset 16
-; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
-; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: .cfi_offset s0, -16
-; RV64-NEXT: mv s0, a0
-; RV64-NEXT: sw a1, 0(a0)
-; RV64-NEXT: call bar
-; RV64-NEXT: lw a0, 0(s0)
-; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64-NEXT: .cfi_restore ra
-; RV64-NEXT: .cfi_restore s0
-; RV64-NEXT: addi sp, sp, 16
-; RV64-NEXT: .cfi_def_cfa_offset 0
-; RV64-NEXT: ret
- store i32 %val, ptr %ptr, align 4
- call void @bar()
- %loaded = load i32, ptr %ptr, align 4
- ret i32 %loaded
-}
-
-declare void @bar()
>From 8d023d1af56870f5d4569a1df9685fe3affc1395 Mon Sep 17 00:00:00 2001
From: Djordje Todorovic <djordje.todorovic at htecgroup.com>
Date: Mon, 4 May 2026 12:34:41 +0200
Subject: [PATCH 5/5] [RISCV] Add FIXME for Zilsd on RV32 in zero-store merge
---
llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
index d20cf4f38ebab..c9ab2d2a81e6e 100644
--- a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -554,6 +554,8 @@ bool RISCVLoadStoreOpt::tryToMergeZeroStInst(
return false;
// Skip if there is no wider store on this target (e.g. SW on RV32).
+ // FIXME: With Zilsd on RV32, two SWs could be merged using SD via a
+ // register pair.
if (!getMatchingWideOpcode(MI.getOpcode(), STI->is64Bit()))
return false;
More information about the llvm-commits
mailing list