[llvm] [DAGCombiner] Forward a narrow load from an overlapping wider load (PR #212667)
Akshay K via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 08:54:45 PDT 2026
https://github.com/kumarak updated https://github.com/llvm/llvm-project/pull/212667
>From e4104e51b35b7216ba0da6e75ad1e8f5c345d354 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 27 Jul 2026 14:54:36 -0400
Subject: [PATCH 1/4] [DAGCombiner] Forward a narrow load from an overlapping
wider load
When a narrow scalar-integer load reads bytes contained in a wider load at an
overlapping address on the same chain, forward it as a truncate of the wider
load (of a shift, when the narrow load sits at a nonzero byte offset) instead of
re-reading memory. SelectionDAG's CSE cannot merge these because they have
different types; such overlapping loads are commonly created by the vectorizer.
Conservative for now: same chain token (so no aliasing store can be sequenced
between the two loads, needing no alias analysis), the narrow load fully
contained in the wider one, non-extending scalar integer loads, little-endian.
The scan of chain users is bounded (combiner-load-forward-max-chain-users) to
stay linear when many loads share one chain token.
Fixes #205978
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 83 ++++++++++++
llvm/test/CodeGen/X86/load-to-load-forward.ll | 118 ++++++++++++++++++
2 files changed, 201 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/load-to-load-forward.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ae9af86196c86..9418d32f4f150 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,6 +138,13 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
cl::desc("Limit the number of times for the same StoreNode and RootNode "
"to bail out in store merging dependence check"));
+// Small cap: a foldable overlapping load has few chain siblings; keeps the
+// scan linear.
+static cl::opt<unsigned> LoadForwardMaxChainUsers(
+ "combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
+ cl::desc("Limit the number of chain users scanned when forwarding a narrow "
+ "load from an overlapping wider load on the same chain"));
+
static cl::opt<bool> EnableReduceLoadOpStoreWidth(
"combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
cl::desc("DAG combiner enable reducing the width of load/op/store "
@@ -385,6 +392,7 @@ namespace {
StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
// Scalars have size 0 to distinguish from singleton vectors.
SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
+ SDValue ForwardLoadValueToDirectLoad(LoadSDNode *LD);
bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
@@ -21864,6 +21872,76 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
return SDValue();
}
+/// If LD reads bytes contained in a wider load from an overlapping address on
+/// the same chain, forward it from that wider load rather than re-reading
+/// memory. This catches overlapping loads of different types (e.g. created by
+/// the vectorizer) that SelectionDAG's CSE cannot merge.
+SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
+ if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+ return SDValue();
+ // Plain non-extending integer loads where the value covers the whole access,
+ // and little-endian so byte offset N maps to bit offset N*8.
+ if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
+ DAG.getDataLayout().isBigEndian())
+ return SDValue();
+ EVT LDVT = LD->getValueType(0);
+ if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
+ LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+ return SDValue();
+
+ SDValue Chain = LD->getChain();
+ BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
+
+ // Bound the scan of chain users: many loads can share one chain token (e.g.
+ // the entry token in a store-free function), which would make this quadratic.
+ unsigned Scanned = 0;
+ for (SDNode *U : Chain.getNode()->users()) {
+ if (++Scanned > LoadForwardMaxChainUsers)
+ break;
+ auto *Wide = dyn_cast<LoadSDNode>(U);
+ if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
+ Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
+ Wide->getAddressSpace() != LD->getAddressSpace())
+ continue;
+
+ EVT WideVT = Wide->getValueType(0);
+ if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
+ WideVT.isVector() || WideVT.isScalableVT() ||
+ WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+ continue;
+
+ // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
+ // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
+ // that keeps LD's start inside Wide and lets the bit math below stay in
+ // range (and avoids negating INT64_MIN).
+ int64_t Off;
+ BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
+ int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
+ if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
+ Off <= -WideBytes)
+ continue;
+ int64_t ByteOff = -Off;
+ // LD must be fully contained, not just start inside Wide.
+ if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+ continue;
+
+ // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
+ // (little-endian) then truncate.
+ SDLoc DL(LD);
+ SDValue Val(Wide, 0);
+ if (ByteOff != 0) {
+ if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
+ continue;
+ Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
+ DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
+ }
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+ // LD performs no write, so its chain successors can use LD's input chain.
+ return CombineTo(LD, Trunc, Chain);
+ }
+ return SDValue();
+}
+
SDValue DAGCombiner::visitLOAD(SDNode *N) {
LoadSDNode *LD = cast<LoadSDNode>(N);
SDValue Chain = LD->getChain();
@@ -21932,6 +22010,11 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
if (auto V = ForwardStoreValueToDirectLoad(LD))
return V;
+ // If this load reads the low bits of a wider load from the same address,
+ // forward it from that load instead of re-reading memory.
+ if (SDValue V = ForwardLoadValueToDirectLoad(LD))
+ return V;
+
// Try to infer better alignment information than the load already has.
if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() &&
!LD->isAtomic()) {
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
new file mode 100644
index 0000000000000..d6452c8e49b30
--- /dev/null
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+; A wide load and a narrow load hit the same address on the same chain (as the
+; SLP vectorizer can produce). The narrow load must be forwarded from the wide
+; load instead of re-reading memory (no second access of d+24), so the cmov
+; takes a register operand.
+
+ at d = external global [0 x i8]
+
+define { i64, i32 } @overlap(i64 %x) nounwind {
+; CHECK-LABEL: overlap:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq 16(%rax), %rcx
+; CHECK-NEXT: movq 24(%rax), %rsi
+; CHECK-NEXT: movq %rcx, (%rax)
+; CHECK-NEXT: movq %rsi, 8(%rax)
+; CHECK-NEXT: testq %rcx, %rcx
+; CHECK-NEXT: movl $3, %eax
+; CHECK-NEXT: cmovneq %rcx, %rax
+; CHECK-NEXT: movl $9, %edx
+; CHECK-NEXT: cmovnel %esi, %edx
+; CHECK-NEXT: retq
+entry:
+ %b = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 16)
+ store i64 %b, ptr @d
+ %wide = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ %c = load i32, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ store <2 x i32> %wide, ptr getelementptr inbounds (i8, ptr @d, i64 8)
+ %cmp = icmp eq i64 %b, 0
+ %rb = select i1 %cmp, i64 3, i64 %b
+ %rc = select i1 %cmp, i32 9, i32 %c
+ %r0 = insertvalue { i64, i32 } poison, i64 %rb, 0
+ %r1 = insertvalue { i64, i32 } %r0, i32 %rc, 1
+ ret { i64, i32 } %r1
+}
+
+; Negative: a store to the same address is sequenced between the loads, so the
+; narrow load reads the new value and must NOT be forwarded from the wide load.
+define i32 @store_between(i64 %v) nounwind {
+; CHECK-LABEL: store_between:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rdi, (%rcx)
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %v, ptr @d
+ %n = load i32, ptr @d
+ ret i32 %n
+}
+
+; A narrow load of a higher field (the top i32 of an i64 load) is forwarded as
+; (trunc (srl %w, 32)) — one load of @q, a shift, no second access.
+ at q = external global i64
+ at s1 = external global i64
+ at s2 = external global i32
+define void @high_field() nounwind {
+; CHECK-LABEL: high_field:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movl %eax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 4
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ store i32 %n, ptr @s2
+ ret void
+}
+
+; Negative: a huge byte offset (2^61) must not overflow the containment check
+; and wrongly forward; the narrow load must be kept.
+define void @huge_offset() nounwind {
+; CHECK-LABEL: huge_offset:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rcx
+; CHECK-NEXT: movabsq $2305843009213693952, %rdx # imm = 0x2000000000000000
+; CHECK-NEXT: movl (%rax,%rdx), %eax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rdx
+; CHECK-NEXT: movq %rcx, (%rdx)
+; CHECK-NEXT: movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movl %eax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 2305843009213693952
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ store i32 %n, ptr @s2
+ ret void
+}
+
+; Negative: a volatile narrow load must be preserved, not forwarded.
+define i32 @volatile_narrow() nounwind {
+; CHECK-LABEL: volatile_narrow:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rcx
+; CHECK-NEXT: movq %rcx, 64(%rax)
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %w, ptr getelementptr inbounds (i8, ptr @d, i64 64)
+ %n = load volatile i32, ptr @d
+ ret i32 %n
+}
>From d119ad7e3c0b34d0a22d4c5acd8071239852d6ee Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Sat, 1 Aug 2026 23:56:29 -0400
Subject: [PATCH 2/4] [DAGCombiner] Restrict overlapping-load forwarding to
safe, free cases
ForwardLoadValueToDirectLoad forwarded a narrow load from an overlapping
wider load on the same chain unconditionally. That caused two problems.
ReduceLoadWidth rewrites a truncated load back into a narrow load, the
exact inverse of this fold, so the two combines undid each other forever.
llc hung on CodeGen/X86/bfloat-calling-conv.ll with -mattr=+sse2,
re-combining one node 30k+ times while allocating new nodes. Require the
wide load to have other users: the load ReduceLoadWidth manufactures has
an otherwise-dead wide source, so this breaks the cycle.
Where narrowing is not free, the truncate (plus a shift for a high field)
can cost more than re-reading memory. It turned a RISC-V byte load into a
zext.b of a word load, changing CodeGen/RISCV/pr148084.ll. Gate the fold
on TLI.isTruncateFree.
X86, AArch64, RISCV and ARM CodeGen: 13921 passed, 0 failures.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 91 +++++++++++--------
1 file changed, 52 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9418d32f4f150..352ca1f27a1ab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,8 +138,7 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
cl::desc("Limit the number of times for the same StoreNode and RootNode "
"to bail out in store merging dependence check"));
-// Small cap: a foldable overlapping load has few chain siblings; keeps the
-// scan linear.
+// A foldable overlapping load has few chain siblings, so a small cap suffices.
static cl::opt<unsigned> LoadForwardMaxChainUsers(
"combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
cl::desc("Limit the number of chain users scanned when forwarding a narrow "
@@ -21872,73 +21871,88 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
return SDValue();
}
+/// Returns true if \p LD is a plain non-extending load of a whole fixed-width
+/// scalar integer.
+static bool isForwardableIntegerLoad(LoadSDNode *LD) {
+ if (!LD->isSimple() || LD->isIndexed() ||
+ LD->getExtensionType() != ISD::NON_EXTLOAD)
+ return false;
+ EVT VT = LD->getValueType(0);
+ return VT == LD->getMemoryVT() && VT.isInteger() && !VT.isVector() &&
+ !VT.isScalableVT();
+}
+
/// If LD reads bytes contained in a wider load from an overlapping address on
-/// the same chain, forward it from that wider load rather than re-reading
-/// memory. This catches overlapping loads of different types (e.g. created by
-/// the vectorizer) that SelectionDAG's CSE cannot merge.
+/// the same chain, forward it from that load instead of re-reading memory.
+/// Catches differently-typed overlapping loads that DAG CSE cannot merge.
SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
- if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+ // Little-endian only, so that byte offset N is bit offset N * 8.
+ if (OptLevel == CodeGenOptLevel::None || DAG.getDataLayout().isBigEndian())
return SDValue();
- // Plain non-extending integer loads where the value covers the whole access,
- // and little-endian so byte offset N maps to bit offset N*8.
- if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
- DAG.getDataLayout().isBigEndian())
- return SDValue();
- EVT LDVT = LD->getValueType(0);
- if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
- LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+ if (!isForwardableIntegerLoad(LD) || LD->getBasePtr().isUndef())
return SDValue();
+ EVT LDVT = LD->getValueType(0);
+ uint64_t LDBits = LDVT.getFixedSizeInBits();
SDValue Chain = LD->getChain();
BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
- // Bound the scan of chain users: many loads can share one chain token (e.g.
- // the entry token in a store-free function), which would make this quadratic.
+ // Many loads can share one chain token (e.g. the entry token), so bound the
+ // scan to keep this from going quadratic.
unsigned Scanned = 0;
for (SDNode *U : Chain.getNode()->users()) {
if (++Scanned > LoadForwardMaxChainUsers)
break;
+
auto *Wide = dyn_cast<LoadSDNode>(U);
- if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
- Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
- Wide->getAddressSpace() != LD->getAddressSpace())
+ if (!Wide || Wide == LD || Wide->getChain() != Chain ||
+ Wide->getAddressSpace() != LD->getAddressSpace() ||
+ !isForwardableIntegerLoad(Wide))
+ continue;
+
+ // Wide must already be needed elsewhere, or this just trades one load for
+ // another and fights ReduceLoadWidth, which narrows the truncate back.
+ if (SDValue(Wide, 0).use_empty())
continue;
EVT WideVT = Wide->getValueType(0);
- if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
- WideVT.isVector() || WideVT.isScalableVT() ||
- WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+ uint64_t WideBits = WideVT.getFixedSizeInBits();
+ if (WideBits <= LDBits)
continue;
- // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
- // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
- // that keeps LD's start inside Wide and lets the bit math below stay in
- // range (and avoids negating INT64_MIN).
+ // Off = Wide's address - LD's address, so LD sits -Off bytes into Wide.
+ // Bounding Off to (-WideBytes, 0] keeps the bit math below in range.
int64_t Off;
BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
Off <= -WideBytes)
continue;
- int64_t ByteOff = -Off;
+
// LD must be fully contained, not just start inside Wide.
- if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+ uint64_t ShiftBits = static_cast<uint64_t>(-Off) * 8;
+ if (ShiftBits + LDBits > WideBits)
+ continue;
+
+ // Narrowing must be free: a truncate, plus a shift for a high field, can
+ // cost more than re-reading memory (e.g. RISC-V, where only i64->i32 is
+ // free).
+ if (!TLI.isTruncateFree(WideVT, LDVT))
+ continue;
+ if (ShiftBits && !TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
continue;
- // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
- // (little-endian) then truncate.
+ // LD is bits [ShiftBits, ShiftBits + LDBits) of Wide.
SDLoc DL(LD);
SDValue Val(Wide, 0);
- if (ByteOff != 0) {
- if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
- continue;
+ if (ShiftBits)
Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
- DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
- }
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+ DAG.getShiftAmountConstant(ShiftBits, WideVT, DL));
+
// LD performs no write, so its chain successors can use LD's input chain.
- return CombineTo(LD, Trunc, Chain);
+ return CombineTo(LD, DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val), Chain);
}
+
return SDValue();
}
@@ -22010,8 +22024,7 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
if (auto V = ForwardStoreValueToDirectLoad(LD))
return V;
- // If this load reads the low bits of a wider load from the same address,
- // forward it from that load instead of re-reading memory.
+ // If this load overlaps a wider load, forward it from that load.
if (SDValue V = ForwardLoadValueToDirectLoad(LD))
return V;
>From 7843fed90ad3340152809841c00f24d833b7296b Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 3 Aug 2026 11:41:10 -0400
Subject: [PATCH 3/4] [DAGCombiner] Forward extending loads from an overlapping
wider load
A load carries two types: the value it produces and the memory it reads.
They differ for an extending load, and requiring NON_EXTLOAD meant the
combine never fired on one. That is the common shape in 64-bit code,
because DAGCombine folds zext/sext of a narrow load into a zextload or
sextload before visitLOAD sees it, so any `unsigned x = s->field;` was
missed.
Drive the containment math from the memory type and re-apply the load's
extension to the forwarded value. The wide load must still be
non-extending, so its value is exactly the bytes it reads.
This also fires on the existing pr80911_vector_load_multiuse test, which
loses a redundant load of the same address and a register spill.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 64 +++++++++++++----
llvm/test/CodeGen/X86/load-combine.ll | 18 +++--
llvm/test/CodeGen/X86/load-to-load-forward.ll | 71 +++++++++++++++++++
3 files changed, 130 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 352ca1f27a1ab..fcadf2fa2dfc0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21874,12 +21874,34 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
/// Returns true if \p LD is a plain non-extending load of a whole fixed-width
/// scalar integer.
static bool isForwardableIntegerLoad(LoadSDNode *LD) {
- if (!LD->isSimple() || LD->isIndexed() ||
- LD->getExtensionType() != ISD::NON_EXTLOAD)
+ if (!LD->isSimple() || LD->isIndexed())
return false;
- EVT VT = LD->getValueType(0);
- return VT == LD->getMemoryVT() && VT.isInteger() && !VT.isVector() &&
- !VT.isScalableVT();
+ EVT MemVT = LD->getMemoryVT();
+ return MemVT.isInteger() && !MemVT.isVector() && !MemVT.isScalableVT();
+}
+
+/// Returns true if \p LD may act as the wider load: it must be non-extending,
+/// so its value is exactly the bytes it reads.
+static bool isForwardableWideLoad(LoadSDNode *LD) {
+ return isForwardableIntegerLoad(LD) &&
+ LD->getExtensionType() == ISD::NON_EXTLOAD &&
+ LD->getValueType(0) == LD->getMemoryVT();
+}
+
+/// Maps a load extension type to the node used to re-apply it to a forwarded
+/// value.
+static unsigned getExtendOpcodeForLoad(ISD::LoadExtType ExtType) {
+ switch (ExtType) {
+ case ISD::ZEXTLOAD:
+ return ISD::ZERO_EXTEND;
+ case ISD::SEXTLOAD:
+ return ISD::SIGN_EXTEND;
+ case ISD::EXTLOAD:
+ return ISD::ANY_EXTEND;
+ case ISD::NON_EXTLOAD:
+ break;
+ }
+ llvm_unreachable("not an extending load");
}
/// If LD reads bytes contained in a wider load from an overlapping address on
@@ -21892,8 +21914,11 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
if (!isForwardableIntegerLoad(LD) || LD->getBasePtr().isUndef())
return SDValue();
+ // LDVT is the value LD produces; LDMemVT is the bytes it actually reads.
+ // They differ when LD is an extending load.
EVT LDVT = LD->getValueType(0);
- uint64_t LDBits = LDVT.getFixedSizeInBits();
+ EVT LDMemVT = LD->getMemoryVT();
+ uint64_t LDMemBits = LDMemVT.getFixedSizeInBits();
SDValue Chain = LD->getChain();
BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
@@ -21907,7 +21932,7 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
auto *Wide = dyn_cast<LoadSDNode>(U);
if (!Wide || Wide == LD || Wide->getChain() != Chain ||
Wide->getAddressSpace() != LD->getAddressSpace() ||
- !isForwardableIntegerLoad(Wide))
+ !isForwardableWideLoad(Wide))
continue;
// Wide must already be needed elsewhere, or this just trades one load for
@@ -21917,7 +21942,7 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
EVT WideVT = Wide->getValueType(0);
uint64_t WideBits = WideVT.getFixedSizeInBits();
- if (WideBits <= LDBits)
+ if (WideBits <= LDMemBits)
continue;
// Off = Wide's address - LD's address, so LD sits -Off bytes into Wide.
@@ -21929,28 +21954,41 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
Off <= -WideBytes)
continue;
- // LD must be fully contained, not just start inside Wide.
+ // LD's access must be fully contained, not just start inside Wide.
uint64_t ShiftBits = static_cast<uint64_t>(-Off) * 8;
- if (ShiftBits + LDBits > WideBits)
+ if (ShiftBits + LDMemBits > WideBits)
continue;
// Narrowing must be free: a truncate, plus a shift for a high field, can
// cost more than re-reading memory (e.g. RISC-V, where only i64->i32 is
// free).
- if (!TLI.isTruncateFree(WideVT, LDVT))
+ if (!TLI.isTruncateFree(WideVT, LDMemVT))
continue;
if (ShiftBits && !TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
continue;
- // LD is bits [ShiftBits, ShiftBits + LDBits) of Wide.
+ // An extending load also needs its extension re-applied to the forwarded
+ // value.
+ ISD::LoadExtType ExtType = LD->getExtensionType();
+ unsigned ExtOpc = 0;
+ if (ExtType != ISD::NON_EXTLOAD) {
+ ExtOpc = getExtendOpcodeForLoad(ExtType);
+ if (LegalOperations && !TLI.isOperationLegalOrCustom(ExtOpc, LDVT))
+ continue;
+ }
+
+ // LD reads bits [ShiftBits, ShiftBits + LDMemBits) of Wide.
SDLoc DL(LD);
SDValue Val(Wide, 0);
if (ShiftBits)
Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
DAG.getShiftAmountConstant(ShiftBits, WideVT, DL));
+ Val = DAG.getNode(ISD::TRUNCATE, DL, LDMemVT, Val);
+ if (ExtOpc)
+ Val = DAG.getNode(ExtOpc, DL, LDVT, Val);
// LD performs no write, so its chain successors can use LD's input chain.
- return CombineTo(LD, DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val), Chain);
+ return CombineTo(LD, Val, Chain);
}
return SDValue();
diff --git a/llvm/test/CodeGen/X86/load-combine.ll b/llvm/test/CodeGen/X86/load-combine.ll
index f21c07599d6f1..3b7025a512e2d 100644
--- a/llvm/test/CodeGen/X86/load-combine.ll
+++ b/llvm/test/CodeGen/X86/load-combine.ll
@@ -1286,22 +1286,20 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(ptr %arg) {
define i32 @pr80911_vector_load_multiuse(ptr %ptr, ptr %clobber) nounwind {
; CHECK-LABEL: pr80911_vector_load_multiuse:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT: movl (%edx), %esi
-; CHECK-NEXT: movzwl (%edx), %eax
-; CHECK-NEXT: movl $0, (%ecx)
-; CHECK-NEXT: movl %esi, (%edx)
-; CHECK-NEXT: popl %esi
+; CHECK-NEXT: movl (%ecx), %edx
+; CHECK-NEXT: movl $0, (%eax)
+; CHECK-NEXT: movl %edx, (%ecx)
+; CHECK-NEXT: movzwl %dx, %eax
; CHECK-NEXT: retl
;
; CHECK64-LABEL: pr80911_vector_load_multiuse:
; CHECK64: # %bb.0:
-; CHECK64-NEXT: movl (%rdi), %ecx
-; CHECK64-NEXT: movzwl (%rdi), %eax
+; CHECK64-NEXT: movl (%rdi), %eax
; CHECK64-NEXT: movl $0, (%rsi)
-; CHECK64-NEXT: movl %ecx, (%rdi)
+; CHECK64-NEXT: movl %eax, (%rdi)
+; CHECK64-NEXT: movzwl %ax, %eax
; CHECK64-NEXT: retq
%load = load <4 x i8>, ptr %ptr, align 16
store i32 0, ptr %clobber
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
index d6452c8e49b30..bce4226757276 100644
--- a/llvm/test/CodeGen/X86/load-to-load-forward.ll
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -116,3 +116,74 @@ entry:
%n = load volatile i32, ptr @d
ret i32 %n
}
+
+ at s3 = external global i64
+
+; The narrow load is zero-extended, so DAGCombine has already folded the zext
+; into a zextload by the time visitLOAD sees it. The forwarded value must have
+; the extension re-applied: (zext (trunc (srl %w, 32))).
+define void @high_field_zext() nounwind {
+; CHECK-LABEL: high_field_zext:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: movq s3 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 4
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ %z = zext i32 %n to i64
+ store i64 %z, ptr @s3
+ ret void
+}
+
+; Same, sign-extended.
+define void @high_field_sext() nounwind {
+; CHECK-LABEL: high_field_sext:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: sarq $32, %rax
+; CHECK-NEXT: movq s3 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 4
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ %z = sext i32 %n to i64
+ store i64 %z, ptr @s3
+ ret void
+}
+
+; A sub-word extending load (i8 out of the middle of an i64).
+define void @byte_field_zext() nounwind {
+; CHECK-LABEL: byte_field_zext:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; CHECK-NEXT: shrl $24, %eax
+; CHECK-NEXT: movq s3 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %bp = getelementptr inbounds i8, ptr @q, i64 3
+ %n = load i8, ptr %bp
+ store i64 %w, ptr @s1
+ %z = zext i8 %n to i64
+ store i64 %z, ptr @s3
+ ret void
+}
>From 22c6ff58bb31d0aa538d2fd0f4be9a6dc68304c3 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 3 Aug 2026 11:53:42 -0400
Subject: [PATCH 4/4] [DAGCombiner] Document cycle safety and pin the overlap
test
Record why the rewrite cannot introduce a cycle. Forwarding makes LD's
users depend on Wide, so Wide must not depend on LD: its chain operand is
LD's chain rather than LD, and equalBaseIndex only matches when Wide's
base and index are the same nodes as LD's, which LD's own address cannot
reference.
The overlap test loads <2 x i32>, which the combine skips, so it only
exercises the transform once type legalization has scalarized that to an
i64 load. Note the dependency, and add overlap_scalar covering the same
shape with an i64 wide load so the coverage does not rest on an earlier
fold. Both emit identical code.
No functional change.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 ++++
llvm/test/CodeGen/X86/load-to-load-forward.ll | 38 +++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index fcadf2fa2dfc0..3fca150d3a362 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21947,6 +21947,13 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
// Off = Wide's address - LD's address, so LD sits -Off bytes into Wide.
// Bounding Off to (-WideBytes, 0] keeps the bit math below in range.
+ //
+ // Matching here is also what makes the rewrite acyclic. Forwarding makes
+ // LD's users depend on Wide, so Wide must not depend on LD. Its chain
+ // operand is LD's chain rather than LD itself, and equalBaseIndex only
+ // succeeds when Wide's base and index are the same nodes as LD's, which
+ // LD's own address cannot reference. So Wide's address cannot reference
+ // LD either.
int64_t Off;
BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
index bce4226757276..12bd9033d1691 100644
--- a/llvm/test/CodeGen/X86/load-to-load-forward.ll
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -5,6 +5,14 @@
; SLP vectorizer can produce). The narrow load must be forwarded from the wide
; load instead of re-reading memory (no second access of d+24), so the cmov
; takes a register operand.
+;
+; Note that the combine ignores vector loads, so it does not act on the
+; <2 x i32> below directly: it fires only after type legalization has turned
+; that into a scalar i64 load. This is the end-to-end case; the scalar tests
+; further down exercise the combine without relying on that. If the checks
+; here ever need regenerating, confirm d+24 is still accessed exactly once
+; rather than just accepting the new output
+; (llc -combiner-load-forward-max-chain-users=0 shows the unforwarded form).
@d = external global [0 x i8]
@@ -187,3 +195,33 @@ entry:
store i64 %z, ptr @s3
ret void
}
+
+; The scalar equivalent of @overlap: same shape, but the wide load is already
+; an i64 so the combine applies without depending on type legalization.
+define { i64, i32 } @overlap_scalar(i64 %x) nounwind {
+; CHECK-LABEL: overlap_scalar:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq 16(%rax), %rcx
+; CHECK-NEXT: movq 24(%rax), %rsi
+; CHECK-NEXT: movq %rcx, (%rax)
+; CHECK-NEXT: movq %rsi, 8(%rax)
+; CHECK-NEXT: testq %rcx, %rcx
+; CHECK-NEXT: movl $3, %eax
+; CHECK-NEXT: cmovneq %rcx, %rax
+; CHECK-NEXT: movl $9, %edx
+; CHECK-NEXT: cmovnel %esi, %edx
+; CHECK-NEXT: retq
+entry:
+ %b = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 16)
+ store i64 %b, ptr @d
+ %wide = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ %c = load i32, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ store i64 %wide, ptr getelementptr inbounds (i8, ptr @d, i64 8)
+ %cmp = icmp eq i64 %b, 0
+ %rb = select i1 %cmp, i64 3, i64 %b
+ %rc = select i1 %cmp, i32 9, i32 %c
+ %r0 = insertvalue { i64, i32 } poison, i64 %rb, 0
+ %r1 = insertvalue { i64, i32 } %r0, i32 %rc, 1
+ ret { i64, i32 } %r1
+}
More information about the llvm-commits
mailing list