[llvm] [DAGCombiner] Forward a narrow load from an overlapping wider load (PR #212667)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 18:53:43 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Akshay K (kumarak)
<details>
<summary>Changes</summary>
## Summary
SelectionDAG can contain overlapping loads from the same address with different value types, such as a vector load introduced by SLP vectorization alongside an existing scalar load. Since these loads have different value types, they cannot be eliminated by the existing DAG CSE, resulting in redundant memory accesses.
This PR fixes #<!-- -->205978 by introducing a new DAG combine, `ForwardLoadValueToDirectLoad`, which forwards a narrow integer load from a wider overlapping load on the same memory chain when it is safe and legal to do so.
### Changes
- Added `ForwardLoadValueToDirectLoad`, a DAG combine that forwards a narrow integer load from a wider overlapping load on the same memory chain.
- Invoked the combine from `visitLOAD` before other load combines.
- Added the `combiner-load-forward-max-chain-users` command-line option to bound the number of chain users examined during traversal, avoiding pathological compile-time behavior.
- Added unit tests covering both forwarding and non-forwarding cases, including intervening stores and volatile loads.
Fixes #<!-- -->205978
Assisted-by: Cursor
---
Full diff: https://github.com/llvm/llvm-project/pull/212667.diff
2 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+79)
- (added) llvm/test/CodeGen/X86/load-to-load-forward.ll (+95)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 232cd609251db..a63fa14a25dcf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,6 +138,13 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
cl::desc("Limit the number of times for the same StoreNode and RootNode "
"to bail out in store merging dependence check"));
+// Small cap: a foldable overlapping load has few chain siblings; keeps the
+// scan linear.
+static cl::opt<unsigned> LoadForwardMaxChainUsers(
+ "combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
+ cl::desc("Limit the number of chain users scanned when forwarding a narrow "
+ "load from an overlapping wider load on the same chain"));
+
static cl::opt<bool> EnableReduceLoadOpStoreWidth(
"combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
cl::desc("DAG combiner enable reducing the width of load/op/store "
@@ -385,6 +392,7 @@ namespace {
StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
// Scalars have size 0 to distinguish from singleton vectors.
SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
+ SDValue ForwardLoadValueToDirectLoad(LoadSDNode *LD);
bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
@@ -21856,6 +21864,72 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
return SDValue();
}
+/// If LD reads bytes contained in a wider load from an overlapping address on
+/// the same chain, forward it from that wider load rather than re-reading
+/// memory. This catches overlapping loads of different types (e.g. created by
+/// the vectorizer) that SelectionDAG's CSE cannot merge.
+SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
+ if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+ return SDValue();
+ // Plain non-extending integer loads where the value covers the whole access,
+ // and little-endian so byte offset N maps to bit offset N*8.
+ if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
+ DAG.getDataLayout().isBigEndian())
+ return SDValue();
+ EVT LDVT = LD->getValueType(0);
+ if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
+ LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+ return SDValue();
+
+ SDValue Chain = LD->getChain();
+ BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
+
+ // Bound the scan of chain users: many loads can share one chain token (e.g.
+ // the entry token in a store-free function), which would make this quadratic.
+ unsigned Scanned = 0;
+ for (SDNode *U : Chain.getNode()->users()) {
+ if (++Scanned > LoadForwardMaxChainUsers)
+ break;
+ auto *Wide = dyn_cast<LoadSDNode>(U);
+ if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
+ Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
+ Wide->getAddressSpace() != LD->getAddressSpace())
+ continue;
+
+ EVT WideVT = Wide->getValueType(0);
+ if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
+ WideVT.isVector() || WideVT.isScalableVT() ||
+ WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+ continue;
+
+ // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
+ // ByteOff = -Off bytes into Wide. Require LD to be fully contained.
+ int64_t Off;
+ BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
+ if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off))
+ continue;
+ int64_t ByteOff = -Off;
+ if (ByteOff < 0 ||
+ ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+ continue;
+
+ // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
+ // (little-endian) then truncate.
+ SDLoc DL(LD);
+ SDValue Val(Wide, 0);
+ if (ByteOff != 0) {
+ if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
+ continue;
+ Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
+ DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
+ }
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+ // LD performs no write, so its chain successors can use LD's input chain.
+ return CombineTo(LD, Trunc, Chain);
+ }
+ return SDValue();
+}
+
SDValue DAGCombiner::visitLOAD(SDNode *N) {
LoadSDNode *LD = cast<LoadSDNode>(N);
SDValue Chain = LD->getChain();
@@ -21924,6 +21998,11 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
if (auto V = ForwardStoreValueToDirectLoad(LD))
return V;
+ // If this load reads the low bits of a wider load from the same address,
+ // forward it from that load instead of re-reading memory.
+ if (SDValue V = ForwardLoadValueToDirectLoad(LD))
+ return V;
+
// Try to infer better alignment information than the load already has.
if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() &&
!LD->isAtomic()) {
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
new file mode 100644
index 0000000000000..b58190af656c0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+; A wide load and a narrow load hit the same address on the same chain (as the
+; SLP vectorizer can produce). The narrow load must be forwarded from the wide
+; load instead of re-reading memory (no second access of d+24), so the cmov
+; takes a register operand.
+
+ at d = external global [0 x i8]
+
+define { i64, i32 } @overlap(i64 %x) nounwind {
+; CHECK-LABEL: overlap:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq 16(%rax), %rcx
+; CHECK-NEXT: movq 24(%rax), %rsi
+; CHECK-NEXT: movq %rcx, (%rax)
+; CHECK-NEXT: movq %rsi, 8(%rax)
+; CHECK-NEXT: testq %rcx, %rcx
+; CHECK-NEXT: movl $3, %eax
+; CHECK-NEXT: cmovneq %rcx, %rax
+; CHECK-NEXT: movl $9, %edx
+; CHECK-NEXT: cmovnel %esi, %edx
+; CHECK-NEXT: retq
+entry:
+ %b = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 16)
+ store i64 %b, ptr @d
+ %wide = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ %c = load i32, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+ store <2 x i32> %wide, ptr getelementptr inbounds (i8, ptr @d, i64 8)
+ %cmp = icmp eq i64 %b, 0
+ %rb = select i1 %cmp, i64 3, i64 %b
+ %rc = select i1 %cmp, i32 9, i32 %c
+ %r0 = insertvalue { i64, i32 } poison, i64 %rb, 0
+ %r1 = insertvalue { i64, i32 } %r0, i32 %rc, 1
+ ret { i64, i32 } %r1
+}
+
+; Negative: a store to the same address is sequenced between the loads, so the
+; narrow load reads the new value and must NOT be forwarded from the wide load.
+define i32 @store_between(i64 %v) nounwind {
+; CHECK-LABEL: store_between:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rdi, (%rcx)
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %v, ptr @d
+ %n = load i32, ptr @d
+ ret i32 %n
+}
+
+; A narrow load of a higher field (the top i32 of an i64 load) is forwarded as
+; (trunc (srl %w, 32)) — one load of @q, a shift, no second access.
+ at q = external global i64
+ at s1 = external global i64
+ at s2 = external global i32
+define void @high_field() nounwind {
+; CHECK-LABEL: high_field:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rax
+; CHECK-NEXT: movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movq %rax, (%rcx)
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT: movl %eax, (%rcx)
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @q
+ %hp = getelementptr inbounds i8, ptr @q, i64 4
+ %n = load i32, ptr %hp
+ store i64 %w, ptr @s1
+ store i32 %n, ptr @s2
+ ret void
+}
+
+; Negative: a volatile narrow load must be preserved, not forwarded.
+define i32 @volatile_narrow() nounwind {
+; CHECK-LABEL: volatile_narrow:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT: movq (%rax), %rcx
+; CHECK-NEXT: movq %rcx, 64(%rax)
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: retq
+entry:
+ %w = load i64, ptr @d
+ store i64 %w, ptr getelementptr inbounds (i8, ptr @d, i64 64)
+ %n = load volatile i32, ptr @d
+ ret i32 %n
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/212667
More information about the llvm-commits
mailing list