[llvm] [DAGCombiner] Forward a narrow load from an overlapping wider load (PR #212667)
Akshay K via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 07:56:11 PDT 2026
https://github.com/kumarak updated https://github.com/llvm/llvm-project/pull/212667
>From e4104e51b35b7216ba0da6e75ad1e8f5c345d354 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 27 Jul 2026 14:54:36 -0400
Subject: [PATCH 1/2] [DAGCombiner] Forward a narrow load from an overlapping
wider load
When a narrow scalar-integer load reads bytes contained in a wider load at an
overlapping address on the same chain, forward it as a truncate of the wider
load (of a shift, when the narrow load sits at a nonzero byte offset) instead of
re-reading memory. SelectionDAG's CSE cannot merge these because they have
different types; such overlapping loads are commonly created by the vectorizer.
Conservative for now: same chain token (so no aliasing store can be sequenced
between the two loads, needing no alias analysis), the narrow load fully
contained in the wider one, non-extending scalar integer loads, little-endian.
The scan of chain users is bounded (combiner-load-forward-max-chain-users) to
stay linear when many loads share one chain token.
Fixes #205978
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 83 ++++++++++++
llvm/test/CodeGen/X86/load-to-load-forward.ll | 118 ++++++++++++++++++
2 files changed, 201 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/load-to-load-forward.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ae9af86196c86..9418d32f4f150 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,6 +138,13 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
cl::desc("Limit the number of times for the same StoreNode and RootNode "
"to bail out in store merging dependence check"));
+// Small cap: a foldable overlapping load has few chain siblings; keeps the
+// scan linear.
+static cl::opt<unsigned> LoadForwardMaxChainUsers(
+ "combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
+ cl::desc("Limit the number of chain users scanned when forwarding a narrow "
+ "load from an overlapping wider load on the same chain"));
+
static cl::opt<bool> EnableReduceLoadOpStoreWidth(
"combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
cl::desc("DAG combiner enable reducing the width of load/op/store "
@@ -385,6 +392,7 @@ namespace {
StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
// Scalars have size 0 to distinguish from singleton vectors.
SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
+ SDValue ForwardLoadValueToDirectLoad(LoadSDNode *LD);
bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
@@ -21864,6 +21872,76 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
return SDValue();
}
+/// If LD reads bytes contained in a wider load from an overlapping address on
+/// the same chain, forward it from that wider load rather than re-reading
+/// memory. This catches overlapping loads of different types (e.g. created by
+/// the vectorizer) that SelectionDAG's CSE cannot merge.
+SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
+ if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+ return SDValue();
+ // Plain non-extending integer loads where the value covers the whole access,
+ // and little-endian so byte offset N maps to bit offset N*8.
+ if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
+ DAG.getDataLayout().isBigEndian())
+ return SDValue();
+ EVT LDVT = LD->getValueType(0);
+ if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
+ LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+ return SDValue();
+
+ SDValue Chain = LD->getChain();
+ BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
+
+ // Bound the scan of chain users: many loads can share one chain token (e.g.
+ // the entry token in a store-free function), which would make this quadratic.
+ unsigned Scanned = 0;
+ for (SDNode *U : Chain.getNode()->users()) {
+ if (++Scanned > LoadForwardMaxChainUsers)
+ break;
+ auto *Wide = dyn_cast<LoadSDNode>(U);
+ if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
+ Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
+ Wide->getAddressSpace() != LD->getAddressSpace())
+ continue;
+
+ EVT WideVT = Wide->getValueType(0);
+ if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
+ WideVT.isVector() || WideVT.isScalableVT() ||
+ WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+ continue;
+
+ // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
+ // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
+ // that keeps LD's start inside Wide and lets the bit math below stay in
+ // range (and avoids negating INT64_MIN).
+ int64_t Off;
+ BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
+ int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
+ if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
+ Off <= -WideBytes)
+ continue;
+ int64_t ByteOff = -Off;
+ // LD must be fully contained, not just start inside Wide.
+ if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+ continue;
+
+ // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
+ // (little-endian) then truncate.
+ SDLoc DL(LD);
+ SDValue Val(Wide, 0);
+ if (ByteOff != 0) {
+ if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
+ continue;
+ Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
+ DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
+ }
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+ // LD performs no write, so its chain successors can use LD's input chain.
+ return CombineTo(LD, Trunc, Chain);
+ }
+ return SDValue();
+}
+
SDValue DAGCombiner::visitLOAD(SDNode *N) {
LoadSDNode *LD = cast<LoadSDNode>(N);
SDValue Chain = LD->getChain();
@@ -21932,6 +22010,11 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
if (auto V = ForwardStoreValueToDirectLoad(LD))
return V;
+ // If this load reads the low bits of a wider load from the same address,
+ // forward it from that load instead of re-reading memory.
+ if (SDValue V = ForwardLoadValueToDirectLoad(LD))
+ return V;
+
// Try to infer better alignment information than the load already has.
if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() &&
!LD->isAtomic()) {
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
new file mode 100644
index 0000000000000..d6452c8e49b30
--- /dev/null
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+; A wide load and a narrow load hit the same address on the same chain (as the
+; SLP vectorizer can produce). The narrow load must be forwarded from the wide
+; load instead of re-reading memory (no second access of d+24), so the cmov
+; takes a register operand.
+
+ at d = external global [0 x i8]
+
+define { i64, i32 } @overlap(i64 %x) nounwind {
+; CHECK-LABEL: overlap:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq 16(%rax), %rcx
+; CHECK-NEXT: movq 24(%rax), %rsi
+; CHECK-NEXT: movq %rcx, (%rax)
+; CHECK-NEXT: movq %rsi, 8(%rax)
+; CHECK-NEXT: testq %rcx, %rcx
+; CHECK-NEXT: movl $3, %eax
+; CHECK-NEXT: cmovneq %rcx, %rax
+; CHECK-NEXT: movl $9, %edx
+; CHECK-NEXT: cmovnel %esi, %edx
+; CHECK-NEXT: retq
+entry:
+ %b = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 16)
+ store i64 %b, ptr @d
+ %wide = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ %c = load i32, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ store <2 x i32> %wide, ptr getelementptr inbounds (i8, ptr @d, i64 8)
+ %cmp = icmp eq i64 %b, 0
+ %rb = select i1 %cmp, i64 3, i64 %b
+ %rc = select i1 %cmp, i32 9, i32 %c
+ %r0 = insertvalue { i64, i32 } poison, i64 %rb, 0
+ %r1 = insertvalue { i64, i32 } %r0, i32 %rc, 1
+ ret { i64, i32 } %r1
+}
+
+; Negative: a store to the same address is sequenced between the loads, so the
+; narrow load reads the new value and must NOT be forwarded from the wide load.
+define i32 @store_between(i64 %v) nounwind {
+; CHECK-LABEL: store_between:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rdi, (%rcx)
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %v, ptr @d
+ %n = load i32, ptr @d
+ ret i32 %n
+}
+
+; A narrow load of a higher field (the top i32 of an i64 load) is forwarded as
+; (trunc (srl %w, 32)) — one load of @q, a shift, no second access.
+ at q = external global i64
+ at s1 = external global i64
+ at s2 = external global i32
+define void @high_field() nounwind {
+; CHECK-LABEL: high_field:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movl %eax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 4
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ store i32 %n, ptr @s2
+ ret void
+}
+
+; Negative: a huge byte offset (2^61) must not overflow the containment check
+; and wrongly forward; the narrow load must be kept.
+define void @huge_offset() nounwind {
+; CHECK-LABEL: huge_offset:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rcx
+; CHECK-NEXT: movabsq $2305843009213693952, %rdx # imm = 0x2000000000000000
+; CHECK-NEXT: movl (%rax,%rdx), %eax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rdx
+; CHECK-NEXT: movq %rcx, (%rdx)
+; CHECK-NEXT: movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movl %eax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 2305843009213693952
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ store i32 %n, ptr @s2
+ ret void
+}
+
+; Negative: a volatile narrow load must be preserved, not forwarded.
+define i32 @volatile_narrow() nounwind {
+; CHECK-LABEL: volatile_narrow:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rcx
+; CHECK-NEXT: movq %rcx, 64(%rax)
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %w, ptr getelementptr inbounds (i8, ptr @d, i64 64)
+ %n = load volatile i32, ptr @d
+ ret i32 %n
+}
>From d119ad7e3c0b34d0a22d4c5acd8071239852d6ee Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Sat, 1 Aug 2026 23:56:29 -0400
Subject: [PATCH 2/2] [DAGCombiner] Restrict overlapping-load forwarding to
safe, free cases
ForwardLoadValueToDirectLoad forwarded a narrow load from an overlapping
wider load on the same chain unconditionally. That caused two problems.
ReduceLoadWidth rewrites a truncated load back into a narrow load, the
exact inverse of this fold, so the two combines undid each other forever.
llc hung on CodeGen/X86/bfloat-calling-conv.ll with -mattr=+sse2,
re-combining one node 30k+ times while allocating new nodes. Require the
wide load to have other users: the load ReduceLoadWidth manufactures has
an otherwise-dead wide source, so this breaks the cycle.
Where narrowing is not free, the truncate (plus a shift for a high field)
can cost more than re-reading memory. It turned a RISC-V byte load into a
zext.b of a word load, changing CodeGen/RISCV/pr148084.ll. Gate the fold
on TLI.isTruncateFree.
X86, AArch64, RISCV and ARM CodeGen: 13921 passed, 0 failures.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 91 +++++++++++--------
1 file changed, 52 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9418d32f4f150..352ca1f27a1ab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,8 +138,7 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
cl::desc("Limit the number of times for the same StoreNode and RootNode "
"to bail out in store merging dependence check"));
-// Small cap: a foldable overlapping load has few chain siblings; keeps the
-// scan linear.
+// A foldable overlapping load has few chain siblings, so a small cap suffices.
static cl::opt<unsigned> LoadForwardMaxChainUsers(
"combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
cl::desc("Limit the number of chain users scanned when forwarding a narrow "
@@ -21872,73 +21871,88 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
return SDValue();
}
+/// Returns true if \p LD is a plain non-extending load of a whole fixed-width
+/// scalar integer.
+static bool isForwardableIntegerLoad(LoadSDNode *LD) {
+ if (!LD->isSimple() || LD->isIndexed() ||
+ LD->getExtensionType() != ISD::NON_EXTLOAD)
+ return false;
+ EVT VT = LD->getValueType(0);
+ return VT == LD->getMemoryVT() && VT.isInteger() && !VT.isVector() &&
+ !VT.isScalableVT();
+}
+
/// If LD reads bytes contained in a wider load from an overlapping address on
-/// the same chain, forward it from that wider load rather than re-reading
-/// memory. This catches overlapping loads of different types (e.g. created by
-/// the vectorizer) that SelectionDAG's CSE cannot merge.
+/// the same chain, forward it from that load instead of re-reading memory.
+/// Catches differently-typed overlapping loads that DAG CSE cannot merge.
SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
- if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+ // Little-endian only, so that byte offset N is bit offset N * 8.
+ if (OptLevel == CodeGenOptLevel::None || DAG.getDataLayout().isBigEndian())
return SDValue();
- // Plain non-extending integer loads where the value covers the whole access,
- // and little-endian so byte offset N maps to bit offset N*8.
- if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
- DAG.getDataLayout().isBigEndian())
- return SDValue();
- EVT LDVT = LD->getValueType(0);
- if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
- LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+ if (!isForwardableIntegerLoad(LD) || LD->getBasePtr().isUndef())
return SDValue();
+ EVT LDVT = LD->getValueType(0);
+ uint64_t LDBits = LDVT.getFixedSizeInBits();
SDValue Chain = LD->getChain();
BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
- // Bound the scan of chain users: many loads can share one chain token (e.g.
- // the entry token in a store-free function), which would make this quadratic.
+ // Many loads can share one chain token (e.g. the entry token), so bound the
+ // scan to keep this from going quadratic.
unsigned Scanned = 0;
for (SDNode *U : Chain.getNode()->users()) {
if (++Scanned > LoadForwardMaxChainUsers)
break;
+
auto *Wide = dyn_cast<LoadSDNode>(U);
- if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
- Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
- Wide->getAddressSpace() != LD->getAddressSpace())
+ if (!Wide || Wide == LD || Wide->getChain() != Chain ||
+ Wide->getAddressSpace() != LD->getAddressSpace() ||
+ !isForwardableIntegerLoad(Wide))
+ continue;
+
+ // Wide must already be needed elsewhere, or this just trades one load for
+ // another and fights ReduceLoadWidth, which narrows the truncate back.
+ if (SDValue(Wide, 0).use_empty())
continue;
EVT WideVT = Wide->getValueType(0);
- if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
- WideVT.isVector() || WideVT.isScalableVT() ||
- WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+ uint64_t WideBits = WideVT.getFixedSizeInBits();
+ if (WideBits <= LDBits)
continue;
- // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
- // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
- // that keeps LD's start inside Wide and lets the bit math below stay in
- // range (and avoids negating INT64_MIN).
+ // Off = Wide's address - LD's address, so LD sits -Off bytes into Wide.
+ // Bounding Off to (-WideBytes, 0] keeps the bit math below in range.
int64_t Off;
BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
Off <= -WideBytes)
continue;
- int64_t ByteOff = -Off;
+
// LD must be fully contained, not just start inside Wide.
- if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+ uint64_t ShiftBits = static_cast<uint64_t>(-Off) * 8;
+ if (ShiftBits + LDBits > WideBits)
+ continue;
+
+ // Narrowing must be free: a truncate, plus a shift for a high field, can
+ // cost more than re-reading memory (e.g. RISC-V, where only i64->i32 is
+ // free).
+ if (!TLI.isTruncateFree(WideVT, LDVT))
+ continue;
+ if (ShiftBits && !TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
continue;
- // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
- // (little-endian) then truncate.
+ // LD is bits [ShiftBits, ShiftBits + LDBits) of Wide.
SDLoc DL(LD);
SDValue Val(Wide, 0);
- if (ByteOff != 0) {
- if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
- continue;
+ if (ShiftBits)
Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
- DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
- }
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+ DAG.getShiftAmountConstant(ShiftBits, WideVT, DL));
+
// LD performs no write, so its chain successors can use LD's input chain.
- return CombineTo(LD, Trunc, Chain);
+ return CombineTo(LD, DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val), Chain);
}
+
return SDValue();
}
@@ -22010,8 +22024,7 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
if (auto V = ForwardStoreValueToDirectLoad(LD))
return V;
- // If this load reads the low bits of a wider load from the same address,
- // forward it from that load instead of re-reading memory.
+ // If this load overlaps a wider load, forward it from that load.
if (SDValue V = ForwardLoadValueToDirectLoad(LD))
return V;
More information about the llvm-commits
mailing list