[llvm] [DAGCombiner] Forward a narrow load from an overlapping wider load (PR #212667)

Akshay K via llvm-commits llvm-commits at lists.llvm.org
Mon Aug 3 07:54:43 PDT 2026


https://github.com/kumarak updated https://github.com/llvm/llvm-project/pull/212667

>From 15275a6860980a87692a0c598a2d204ee8154d9d Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 27 Jul 2026 14:54:36 -0400
Subject: [PATCH 1/3] [DAGCombiner] Forward a narrow load from an overlapping
 wider load

When a narrow scalar-integer load reads bytes contained in a wider load at an
overlapping address on the same chain, forward it as a truncate of the wider
load (of a shift, when the narrow load sits at a nonzero byte offset) instead of
re-reading memory. SelectionDAG's CSE cannot merge these because they have
different types; such overlapping loads are commonly created by the vectorizer.

Conservative for now: same chain token (so no aliasing store can be sequenced
between the two loads, needing no alias analysis), the narrow load fully
contained in the wider one, non-extending scalar integer loads, little-endian.
The scan of chain users is bounded (combiner-load-forward-max-chain-users) to
stay linear when many loads share one chain token.

Fixes #205978
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  83 ++++++++++++
 llvm/test/CodeGen/X86/load-to-load-forward.ll | 118 ++++++++++++++++++
 2 files changed, 201 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/load-to-load-forward.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 06c9233d2f497..aa47e77a7a541 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,6 +138,13 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
     cl::desc("Limit the number of times for the same StoreNode and RootNode "
              "to bail out in store merging dependence check"));
 
+// Small cap: a foldable overlapping load has few chain siblings; keeps the
+// scan linear.
+static cl::opt<unsigned> LoadForwardMaxChainUsers(
+    "combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
+    cl::desc("Limit the number of chain users scanned when forwarding a narrow "
+             "load from an overlapping wider load on the same chain"));
+
 static cl::opt<bool> EnableReduceLoadOpStoreWidth(
     "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
     cl::desc("DAG combiner enable reducing the width of load/op/store "
@@ -385,6 +392,7 @@ namespace {
     StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
     // Scalars have size 0 to distinguish from singleton vectors.
     SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
+    SDValue ForwardLoadValueToDirectLoad(LoadSDNode *LD);
     bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
     bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
 
@@ -21864,6 +21872,76 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
   return SDValue();
 }
 
+/// If LD reads bytes contained in a wider load from an overlapping address on
+/// the same chain, forward it from that wider load rather than re-reading
+/// memory. This catches overlapping loads of different types (e.g. created by
+/// the vectorizer) that SelectionDAG's CSE cannot merge.
+SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
+  if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+    return SDValue();
+  // Plain non-extending integer loads where the value covers the whole access,
+  // and little-endian so byte offset N maps to bit offset N*8.
+  if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
+      DAG.getDataLayout().isBigEndian())
+    return SDValue();
+  EVT LDVT = LD->getValueType(0);
+  if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
+      LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+    return SDValue();
+
+  SDValue Chain = LD->getChain();
+  BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
+
+  // Bound the scan of chain users: many loads can share one chain token (e.g.
+  // the entry token in a store-free function), which would make this quadratic.
+  unsigned Scanned = 0;
+  for (SDNode *U : Chain.getNode()->users()) {
+    if (++Scanned > LoadForwardMaxChainUsers)
+      break;
+    auto *Wide = dyn_cast<LoadSDNode>(U);
+    if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
+        Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
+        Wide->getAddressSpace() != LD->getAddressSpace())
+      continue;
+
+    EVT WideVT = Wide->getValueType(0);
+    if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
+        WideVT.isVector() || WideVT.isScalableVT() ||
+        WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+      continue;
+
+    // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
+    // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
+    // that keeps LD's start inside Wide and lets the bit math below stay in
+    // range (and avoids negating INT64_MIN).
+    int64_t Off;
+    BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
+    int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
+    if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
+        Off <= -WideBytes)
+      continue;
+    int64_t ByteOff = -Off;
+    // LD must be fully contained, not just start inside Wide.
+    if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+      continue;
+
+    // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
+    // (little-endian) then truncate.
+    SDLoc DL(LD);
+    SDValue Val(Wide, 0);
+    if (ByteOff != 0) {
+      if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
+        continue;
+      Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
+                        DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
+    }
+    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+    // LD performs no write, so its chain successors can use LD's input chain.
+    return CombineTo(LD, Trunc, Chain);
+  }
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitLOAD(SDNode *N) {
   LoadSDNode *LD  = cast<LoadSDNode>(N);
   SDValue Chain = LD->getChain();
@@ -21932,6 +22010,11 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
   if (auto V = ForwardStoreValueToDirectLoad(LD))
     return V;
 
+  // If this load reads the low bits of a wider load from the same address,
+  // forward it from that load instead of re-reading memory.
+  if (SDValue V = ForwardLoadValueToDirectLoad(LD))
+    return V;
+
   // Try to infer better alignment information than the load already has.
   if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() &&
       !LD->isAtomic()) {
diff --git a/llvm/test/CodeGen/X86/load-to-load-forward.ll b/llvm/test/CodeGen/X86/load-to-load-forward.ll
new file mode 100644
index 0000000000000..d6452c8e49b30
--- /dev/null
+++ b/llvm/test/CodeGen/X86/load-to-load-forward.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+; A wide load and a narrow load hit the same address on the same chain (as the
+; SLP vectorizer can produce). The narrow load must be forwarded from the wide
+; load instead of re-reading memory (no second access of d+24), so the cmov
+; takes a register operand.
+
+ at d = external global [0 x i8]
+
+define { i64, i32 } @overlap(i64 %x) nounwind {
+; CHECK-LABEL: overlap:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT:    movq 16(%rax), %rcx
+; CHECK-NEXT:    movq 24(%rax), %rsi
+; CHECK-NEXT:    movq %rcx, (%rax)
+; CHECK-NEXT:    movq %rsi, 8(%rax)
+; CHECK-NEXT:    testq %rcx, %rcx
+; CHECK-NEXT:    movl $3, %eax
+; CHECK-NEXT:    cmovneq %rcx, %rax
+; CHECK-NEXT:    movl $9, %edx
+; CHECK-NEXT:    cmovnel %esi, %edx
+; CHECK-NEXT:    retq
+entry:
+  %b = load i64, ptr getelementptr inbounds (i8, ptr @d, i64 16)
+  store i64 %b, ptr @d
+  %wide = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+  %c = load i32, ptr getelementptr inbounds (i8, ptr @d, i64 24)
+  store <2 x i32> %wide, ptr getelementptr inbounds (i8, ptr @d, i64 8)
+  %cmp = icmp eq i64 %b, 0
+  %rb = select i1 %cmp, i64 3, i64 %b
+  %rc = select i1 %cmp, i32 9, i32 %c
+  %r0 = insertvalue { i64, i32 } poison, i64 %rb, 0
+  %r1 = insertvalue { i64, i32 } %r0, i32 %rc, 1
+  ret { i64, i32 } %r1
+}
+
+; Negative: a store to the same address is sequenced between the loads, so the
+; narrow load reads the new value and must NOT be forwarded from the wide load.
+define i32 @store_between(i64 %v) nounwind {
+; CHECK-LABEL: store_between:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    movq d at GOTPCREL(%rip), %rcx
+; CHECK-NEXT:    movq %rdi, (%rcx)
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %w = load i64, ptr @d
+  store i64 %v, ptr @d
+  %n = load i32, ptr @d
+  ret i32 %n
+}
+
+; A narrow load of a higher field (the top i32 of an i64 load) is forwarded as
+; (trunc (srl %w, 32)) — one load of @q, a shift, no second access.
+ at q = external global i64
+ at s1 = external global i64
+ at s2 = external global i32
+define void @high_field() nounwind {
+; CHECK-LABEL: high_field:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movq s1 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT:    movq %rax, (%rcx)
+; CHECK-NEXT:    shrq $32, %rax
+; CHECK-NEXT:    movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT:    movl %eax, (%rcx)
+; CHECK-NEXT:    retq
+entry:
+  %w = load i64, ptr @q
+  %hp = getelementptr inbounds i8, ptr @q, i64 4
+  %n = load i32, ptr %hp
+  store i64 %w, ptr @s1
+  store i32 %n, ptr @s2
+  ret void
+}
+
+; Negative: a huge byte offset (2^61) must not overflow the containment check
+; and wrongly forward; the narrow load must be kept.
+define void @huge_offset() nounwind {
+; CHECK-LABEL: huge_offset:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq q at GOTPCREL(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rcx
+; CHECK-NEXT:    movabsq $2305843009213693952, %rdx # imm = 0x2000000000000000
+; CHECK-NEXT:    movl (%rax,%rdx), %eax
+; CHECK-NEXT:    movq s1 at GOTPCREL(%rip), %rdx
+; CHECK-NEXT:    movq %rcx, (%rdx)
+; CHECK-NEXT:    movq s2 at GOTPCREL(%rip), %rcx
+; CHECK-NEXT:    movl %eax, (%rcx)
+; CHECK-NEXT:    retq
+entry:
+  %w = load i64, ptr @q
+  %hp = getelementptr inbounds i8, ptr @q, i64 2305843009213693952
+  %n = load i32, ptr %hp
+  store i64 %w, ptr @s1
+  store i32 %n, ptr @s2
+  ret void
+}
+
+; Negative: a volatile narrow load must be preserved, not forwarded.
+define i32 @volatile_narrow() nounwind {
+; CHECK-LABEL: volatile_narrow:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq d at GOTPCREL(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rcx
+; CHECK-NEXT:    movq %rcx, 64(%rax)
+; CHECK-NEXT:    movl (%rax), %eax
+; CHECK-NEXT:    retq
+entry:
+  %w = load i64, ptr @d
+  store i64 %w, ptr getelementptr inbounds (i8, ptr @d, i64 64)
+  %n = load volatile i32, ptr @d
+  ret i32 %n
+}

>From 53b946dc49d68520ab674a9450a5f2a168188e92 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Sat, 1 Aug 2026 23:56:29 -0400
Subject: [PATCH 2/3] [DAGCombiner] Restrict overlapping-load forwarding to
 safe, free cases

ForwardLoadValueToDirectLoad forwarded a narrow load from an overlapping
wider load on the same chain unconditionally. That caused two problems.

ReduceLoadWidth rewrites a truncated load back into a narrow load, the
exact inverse of this fold, so the two combines undid each other forever.
llc hung on CodeGen/X86/bfloat-calling-conv.ll with -mattr=+sse2,
re-combining one node 30k+ times while allocating new nodes. Require the
wide load to have other users: the load ReduceLoadWidth manufactures has
an otherwise-dead wide source, so this breaks the cycle.

Where narrowing is not free, the truncate (plus a shift for a high field)
can cost more than re-reading memory. It turned a RISC-V byte load into a
zext.b of a word load, changing CodeGen/RISCV/pr148084.ll. Gate the fold
on TLI.isTruncateFree.

X86, AArch64, RISCV and ARM CodeGen: 13921 passed, 0 failures.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aa47e77a7a541..ba464b8a49790 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21904,6 +21904,13 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
         Wide->getAddressSpace() != LD->getAddressSpace())
       continue;
 
+    // The wide load must already be needed for something else. Forwarding from
+    // a wide load with no other users would trade one load for another, and it
+    // also fights ReduceLoadWidth, which turns a truncated load back into a
+    // narrow load: the two combines would undo each other forever.
+    if (SDValue(Wide, 0).use_empty())
+      continue;
+
     EVT WideVT = Wide->getValueType(0);
     if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
         WideVT.isVector() || WideVT.isScalableVT() ||
@@ -21925,6 +21932,13 @@ SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
     if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
       continue;
 
+    // Only forward when narrowing Wide to LD's type is free on the target.
+    // Otherwise the truncate (plus a shift for a high field) can cost more than
+    // simply re-reading memory, which would pessimize targets where a sub-width
+    // truncate is not free (e.g. RISC-V, where only i64->i32 is free).
+    if (!TLI.isTruncateFree(WideVT, LDVT))
+      continue;
+
     // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
     // (little-endian) then truncate.
     SDLoc DL(LD);

>From 2c8c55173a1599450f5bb3c08ca46120b3daba89 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Mon, 3 Aug 2026 10:54:07 -0400
Subject: [PATCH 3/3] [NFC][DAGCombiner] Tidy ForwardLoadValueToDirectLoad

Extract the load-shape predicate shared by the narrow and wide loads into
isForwardableIntegerLoad, so the rule is stated once instead of spelled out
twice in different orders.

Compute the shift amount once as ShiftBits instead of recomputing ByteOff * 8
and mixing signed and unsigned arithmetic, and move the ISD::SRL legality
check out of the emission block so every bail-out precedes node creation.

Also trim the comments. No functional change: the order of the legality
checks is unchanged.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 93 +++++++++----------
 1 file changed, 46 insertions(+), 47 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ba464b8a49790..1912ca6e779e5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -138,8 +138,7 @@ static cl::opt<unsigned> StoreMergeDependenceLimit(
     cl::desc("Limit the number of times for the same StoreNode and RootNode "
              "to bail out in store merging dependence check"));
 
-// Small cap: a foldable overlapping load has few chain siblings; keeps the
-// scan linear.
+// A foldable overlapping load has few chain siblings, so a small cap suffices.
 static cl::opt<unsigned> LoadForwardMaxChainUsers(
     "combiner-load-forward-max-chain-users", cl::Hidden, cl::init(16),
     cl::desc("Limit the number of chain users scanned when forwarding a narrow "
@@ -21872,87 +21871,88 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
   return SDValue();
 }
 
+/// Returns true if \p LD is a plain non-extending load of a whole fixed-width
+/// scalar integer.
+static bool isForwardableIntegerLoad(LoadSDNode *LD) {
+  if (!LD->isSimple() || LD->isIndexed() ||
+      LD->getExtensionType() != ISD::NON_EXTLOAD)
+    return false;
+  EVT VT = LD->getValueType(0);
+  return VT == LD->getMemoryVT() && VT.isInteger() && !VT.isVector() &&
+         !VT.isScalableVT();
+}
+
 /// If LD reads bytes contained in a wider load from an overlapping address on
-/// the same chain, forward it from that wider load rather than re-reading
-/// memory. This catches overlapping loads of different types (e.g. created by
-/// the vectorizer) that SelectionDAG's CSE cannot merge.
+/// the same chain, forward it from that load instead of re-reading memory.
+/// Catches differently-typed overlapping loads that DAG CSE cannot merge.
 SDValue DAGCombiner::ForwardLoadValueToDirectLoad(LoadSDNode *LD) {
-  if (OptLevel == CodeGenOptLevel::None || !LD->isSimple() || LD->isIndexed())
+  // Little-endian only, so that byte offset N is bit offset N * 8.
+  if (OptLevel == CodeGenOptLevel::None || DAG.getDataLayout().isBigEndian())
     return SDValue();
-  // Plain non-extending integer loads where the value covers the whole access,
-  // and little-endian so byte offset N maps to bit offset N*8.
-  if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
-      DAG.getDataLayout().isBigEndian())
-    return SDValue();
-  EVT LDVT = LD->getValueType(0);
-  if (LDVT != LD->getMemoryVT() || !LDVT.isInteger() || LDVT.isVector() ||
-      LDVT.isScalableVT() || LD->getBasePtr().isUndef())
+  if (!isForwardableIntegerLoad(LD) || LD->getBasePtr().isUndef())
     return SDValue();
 
+  EVT LDVT = LD->getValueType(0);
+  uint64_t LDBits = LDVT.getFixedSizeInBits();
   SDValue Chain = LD->getChain();
   BaseIndexOffset LDPtr = BaseIndexOffset::match(LD, DAG);
 
-  // Bound the scan of chain users: many loads can share one chain token (e.g.
-  // the entry token in a store-free function), which would make this quadratic.
+  // Many loads can share one chain token (e.g. the entry token), so bound the
+  // scan to keep this from going quadratic.
   unsigned Scanned = 0;
   for (SDNode *U : Chain.getNode()->users()) {
     if (++Scanned > LoadForwardMaxChainUsers)
       break;
+
     auto *Wide = dyn_cast<LoadSDNode>(U);
-    if (!Wide || Wide == LD || Wide->getChain() != Chain || !Wide->isSimple() ||
-        Wide->isIndexed() || Wide->getExtensionType() != ISD::NON_EXTLOAD ||
-        Wide->getAddressSpace() != LD->getAddressSpace())
+    if (!Wide || Wide == LD || Wide->getChain() != Chain ||
+        Wide->getAddressSpace() != LD->getAddressSpace() ||
+        !isForwardableIntegerLoad(Wide))
       continue;
 
-    // The wide load must already be needed for something else. Forwarding from
-    // a wide load with no other users would trade one load for another, and it
-    // also fights ReduceLoadWidth, which turns a truncated load back into a
-    // narrow load: the two combines would undo each other forever.
+    // Wide must already be needed elsewhere, or this just trades one load for
+    // another and fights ReduceLoadWidth, which narrows the truncate back.
     if (SDValue(Wide, 0).use_empty())
       continue;
 
     EVT WideVT = Wide->getValueType(0);
-    if (WideVT != Wide->getMemoryVT() || !WideVT.isInteger() ||
-        WideVT.isVector() || WideVT.isScalableVT() ||
-        WideVT.getFixedSizeInBits() <= LDVT.getFixedSizeInBits())
+    uint64_t WideBits = WideVT.getFixedSizeInBits();
+    if (WideBits <= LDBits)
       continue;
 
-    // equalBaseIndex sets Off = Wide's address - LD's address, so LD sits
-    // ByteOff = -Off bytes into Wide. Bound Off to (-WideBytes, 0] up front:
-    // that keeps LD's start inside Wide and lets the bit math below stay in
-    // range (and avoids negating INT64_MIN).
+    // Off = Wide's address - LD's address, so LD sits -Off bytes into Wide.
+    // Bounding Off to (-WideBytes, 0] keeps the bit math below in range.
     int64_t Off;
     BaseIndexOffset WidePtr = BaseIndexOffset::match(Wide, DAG);
     int64_t WideBytes = WideVT.getStoreSize().getFixedValue();
     if (!LDPtr.equalBaseIndex(WidePtr, DAG, Off) || Off > 0 ||
         Off <= -WideBytes)
       continue;
-    int64_t ByteOff = -Off;
+
     // LD must be fully contained, not just start inside Wide.
-    if (ByteOff * 8 + LDVT.getFixedSizeInBits() > WideVT.getFixedSizeInBits())
+    uint64_t ShiftBits = static_cast<uint64_t>(-Off) * 8;
+    if (ShiftBits + LDBits > WideBits)
       continue;
 
-    // Only forward when narrowing Wide to LD's type is free on the target.
-    // Otherwise the truncate (plus a shift for a high field) can cost more than
-    // simply re-reading memory, which would pessimize targets where a sub-width
-    // truncate is not free (e.g. RISC-V, where only i64->i32 is free).
+    // Narrowing must be free: a truncate, plus a shift for a high field, can
+    // cost more than re-reading memory (e.g. RISC-V, where only i64->i32 is
+    // free).
     if (!TLI.isTruncateFree(WideVT, LDVT))
       continue;
+    if (ShiftBits && !TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
+      continue;
 
-    // LD is bits [ByteOff*8, ByteOff*8 + LDbits) of Wide: shift them down
-    // (little-endian) then truncate.
+    // LD is bits [ShiftBits, ShiftBits + LDBits) of Wide.
     SDLoc DL(LD);
     SDValue Val(Wide, 0);
-    if (ByteOff != 0) {
-      if (!TLI.isOperationLegalOrCustom(ISD::SRL, WideVT))
-        continue;
+    if (ShiftBits)
       Val = DAG.getNode(ISD::SRL, DL, WideVT, Val,
-                        DAG.getShiftAmountConstant(ByteOff * 8, WideVT, DL));
-    }
-    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val);
+                        DAG.getShiftAmountConstant(ShiftBits, WideVT, DL));
+
     // LD performs no write, so its chain successors can use LD's input chain.
-    return CombineTo(LD, Trunc, Chain);
+    return CombineTo(LD, DAG.getNode(ISD::TRUNCATE, DL, LDVT, Val), Chain);
   }
+
   return SDValue();
 }
 
@@ -22024,8 +22024,7 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
   if (auto V = ForwardStoreValueToDirectLoad(LD))
     return V;
 
-  // If this load reads the low bits of a wider load from the same address,
-  // forward it from that load instead of re-reading memory.
+  // If this load overlaps a wider load, forward it from that load.
   if (SDValue V = ForwardLoadValueToDirectLoad(LD))
     return V;
 



More information about the llvm-commits mailing list