[llvm-branch-commits] [llvm] [DAGCombiner] Fix MatchLoadCombine address for non-zero vector offsets (PR #244904)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sat Nov 21 08:16:10 PST 2026


https://github.com/lijinpei-amd created https://github.com/llvm/llvm-project/pull/244904

When the first byte has a non-zero vector element index, it is
discarded by MatchLoadCombine, which miscompiles. Also, MatchLoadCombine
bails out if the byte offset within the scalar type is non-zero.

Fix the mis-compilation by accumulating both the vector element index
and the offset within scalar elements, and adjust the load address and
alignment accordingly. Fix the missed optimization by dropping the
`MemoryByteOffset == 0` requirement.

Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>



>From a0e76e828ef92a6eead518f2fc35da79bd2a17c9 Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Sat, 21 Nov 2026 19:55:25 +0800
Subject: [PATCH 1/2] [DAGCombiner] Precommit tests for MatchLoadCombine
 non-zero offsets

The byte at the lowest address comes from element 1 of the vector load,
but the combined load currently reads from the vector's base address.
In load_i32_from_nonzero_offset, it is the second byte of an i16 load,
which is currently not combined.

Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>
---
 .../AArch64/load-combine-big-endian.ll        | 58 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/load-combine.ll     | 59 +++++++++++++++++++
 2 files changed, 117 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 3b86acc7720e39..7e15bdde9c112d 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -562,3 +562,61 @@ define i16 @load_i16_from_nonzero_offset(ptr %p) {
   %res = or i16 %v1.shl, %v2
   ret i16 %res
 }
+
+; ptr p;
+; ptr p1.i16 = (ptr) p;
+; ptr p3.i16 = (ptr) &p[3];
+; (p1.i16[0] << 24) | ((i32) p[2] << 16) | p3.i16[0]
+;
+; This is an i32 load from p[1].
+; FIXME: Should be a single load from p[1].
+define i32 @load_i32_from_nonzero_offset(ptr %p) {
+; CHECK-LABEL: load_i32_from_nonzero_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrb w8, [x0, #2]
+; CHECK-NEXT:    ldrh w9, [x0]
+; CHECK-NEXT:    ldurh w10, [x0, #3]
+; CHECK-NEXT:    lsl w8, w8, #16
+; CHECK-NEXT:    orr w8, w8, w9, lsl #24
+; CHECK-NEXT:    orr w0, w8, w10
+; CHECK-NEXT:    ret
+  %p2.i8 = getelementptr i8, ptr %p, i64 2
+  %p3.i16 = getelementptr i8, ptr %p, i64 3
+  %v1 = load i16, ptr %p
+  %v2.i8 = load i8, ptr %p2.i8
+  %v3 = load i16, ptr %p3.i16, align 1
+  %v1.ext = zext i16 %v1 to i32
+  %v2 = zext i8 %v2.i8 to i32
+  %v3.ext = zext i16 %v3 to i32
+  %v1.shl = shl i32 %v1.ext, 24
+  %v2.shl = shl i32 %v2, 16
+  %tmp = or i32 %v1.shl, %v2.shl
+  %res = or i32 %tmp, %v3.ext
+  ret i32 %res
+}
+
+; FIXME: Loads from %in instead of %in + 1.
+define void @load_i16_from_vector_elt1(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: load_i16_from_vector_elt1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldr w9, [x0]
+; CHECK-NEXT:    strh w8, [x1]
+; CHECK-NEXT:    str w9, [x2]
+; CHECK-NEXT:    ret
+  %p2 = getelementptr inbounds i8, ptr %in, i64 2
+  %x2 = load i8, ptr %p2, align 1
+  %ld = load <4 x i8>, ptr %in, align 4
+
+  %e1 = extractelement <4 x i8> %ld, i32 1
+
+  %z0 = zext i8 %x2 to i16
+  %z1 = zext i8 %e1 to i16
+
+  %s1 = shl nuw i16 %z1, 8
+  %i = or i16 %s1, %z0
+
+  store i16 %i, ptr %out
+  store <4 x i8> %ld, ptr %p, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index 0bf4fb10630258..6b6f1e62e80c48 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -714,6 +714,64 @@ define void @short_vector_to_i64(ptr %in, ptr %out, ptr %p) {
   ret void
 }
 
+; x1 = bswap(x0[1:2])
+; FIXME: Loads from x0 instead of x0 + 1.
+define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldr w9, [x0]
+; CHECK-NEXT:    rev w8, w8
+; CHECK-NEXT:    lsr w8, w8, #16
+; CHECK-NEXT:    strh w8, [x1]
+; CHECK-NEXT:    str w9, [x2]
+; CHECK-NEXT:    ret
+  %p2 = getelementptr inbounds i8, ptr %in, i64 2
+  %x2 = load i8, ptr %p2, align 1
+  %ld = load <4 x i8>, ptr %in, align 4
+
+  %e1 = extractelement <4 x i8> %ld, i32 1
+
+  %z0 = zext i8 %x2 to i16
+  %z1 = zext i8 %e1 to i16
+
+  %s1 = shl nuw i16 %z1, 8
+  %i = or i16 %s1, %z0
+
+  store i16 %i, ptr %out
+  store <4 x i8> %ld, ptr %p, align 4
+  ret void
+}
+
+; x1 = bswap(x0[1:2]), with x0 + 1 only 1-byte aligned
+; FIXME: Loads from x0 instead of x0 + 1.
+define void @short_vector_elt1_to_i16_bswap_strict_align(ptr %in, ptr %out, ptr %p) #1 {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap_strict_align:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldr w9, [x0]
+; CHECK-NEXT:    rev w8, w8
+; CHECK-NEXT:    lsr w8, w8, #16
+; CHECK-NEXT:    strh w8, [x1]
+; CHECK-NEXT:    str w9, [x2]
+; CHECK-NEXT:    ret
+  %p2 = getelementptr inbounds i8, ptr %in, i64 2
+  %x2 = load i8, ptr %p2, align 1
+  %ld = load <4 x i8>, ptr %in, align 4
+
+  %e1 = extractelement <4 x i8> %ld, i32 1
+
+  %z0 = zext i8 %x2 to i16
+  %z1 = zext i8 %e1 to i16
+
+  %s1 = shl nuw i16 %z1, 8
+  %i = or i16 %s1, %z0
+
+  store i16 %i, ptr %out
+  store <4 x i8> %ld, ptr %p, align 4
+  ret void
+}
+
 ; x1 = x0
 define void @scalable_vector_to_i32(ptr %in, ptr %out, ptr %p) #0 {
 ; CHECK-LABEL: scalable_vector_to_i32:
@@ -892,3 +950,4 @@ define void @scalable_vector_to_i64(ptr %in, ptr %out, ptr %p) #0 {
 }
 
 attributes #0 = { "target-features"="+sve" }
+attributes #1 = { "target-features"="+strict-align" }

>From 1761f88599f2c9736c60dde93e446b855d71013b Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Sat, 21 Nov 2026 19:56:40 +0800
Subject: [PATCH 2/2] [DAGCombiner] Fix MatchLoadCombine address for non-zero
 vector offsets

When the first byte has a non-zero vector element index, it is
discarded by MatchLoadCombine, which miscompiles. Also, MatchLoadCombine
bails out if the byte offset within the scalar type is non-zero.

Fix the mis-compilation by accumulating both the vector element index
and the offset within scalar elements, and adjust the load address and
alignment accordingly. Fix the missed optimization by dropping the
`MemoryByteOffset == 0` requirement.

Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 28 +++++++++++--------
 .../AArch64/load-combine-big-endian.ll        | 18 +++---------
 llvm/test/CodeGen/AArch64/load-combine.ll     | 15 +++++-----
 3 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f96fc90f7f347a..3ebbbe4846a4b8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10298,6 +10298,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
   SmallPtrSet<LoadSDNode *, 8> Loads;
   std::optional<SDByteProvider> FirstByteProvider;
   int64_t FirstOffset = INT64_MAX;
+  uint64_t ByteOffsetInFirstLoad = 0;
 
   // Check if all the bytes of the OR we are looking at are loaded from the same
   // base address. Collect bytes offsets from Base address in ByteOffsets.
@@ -10339,11 +10340,12 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
     // due to different ExtractElement indices. To accurately determine the
     // byte position of an ExtractElement, we offset the base load ptr with
     // the index multiplied by the byte size of each element in the vector.
+    uint64_t ByteOffsetFromVector = 0;
     if (L->getMemoryVT().isVector()) {
       unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
       if (LoadWidthInBit % 8 != 0)
         return SDValue();
-      unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
+      ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
       Ptr.addToOffset(ByteOffsetFromVector);
     }
 
@@ -10361,6 +10363,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
     if (ByteOffsetFromBase < FirstOffset) {
       FirstByteProvider = P;
       FirstOffset = ByteOffsetFromBase;
+      ByteOffsetInFirstLoad = ByteOffsetFromVector + MemoryByteOffset(*P);
     }
 
     Loads.insert(L);
@@ -10388,19 +10391,16 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
 
   assert(FirstByteProvider && "must be set");
 
-  // Ensure that the first byte is loaded from zero offset of the first load.
-  // So the combined value can be loaded from the first load address.
-  if (MemoryByteOffset(*FirstByteProvider) != 0)
-    return SDValue();
   auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
+  Align FirstAlign =
+      commonAlignment(FirstLoad->getAlign(), ByteOffsetInFirstLoad);
 
   // Before legalization we allow introducing loads that are wider than legal,
   // which will later be split into legally sized loads. This enables us to
   // combine, for example, i8 loads forming an i64 into an i64 load, which get
   // then gets split up into couple of i32 loads on 32 bit targets.
   if (LegalOperations &&
-      !TLI.isLoadLegal(VT, MemVT, FirstLoad->getAlign(),
-                       FirstLoad->getAddressSpace(),
+      !TLI.isLoadLegal(VT, MemVT, FirstAlign, FirstLoad->getAddressSpace(),
                        NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, false))
     return SDValue();
 
@@ -10430,16 +10430,22 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
   unsigned Fast = 0;
   bool Allowed =
       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
-                             *FirstLoad->getMemOperand(), &Fast);
+                             FirstLoad->getAddressSpace(), FirstAlign,
+                             FirstLoad->getMemOperand()->getFlags(), &Fast);
   if (!Allowed || !Fast)
     return SDValue();
 
   const MDNode *MemCacheHint = getCommonMemCacheHint(Loads);
 
+  SDValue Ptr = FirstLoad->getBasePtr();
+  if (ByteOffsetInFirstLoad)
+    Ptr = DAG.getMemBasePlusOffset(
+        Ptr, TypeSize::getFixed(ByteOffsetInFirstLoad), SDLoc(N));
+
   SDValue NewLoad = DAG.getExtLoad(
-      NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
-      FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), MemVT,
-      FirstLoad->getAlign(), MachineMemOperand::MONone,
+      NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain, Ptr,
+      FirstLoad->getPointerInfo().getWithOffset(ByteOffsetInFirstLoad), MemVT,
+      FirstAlign, MachineMemOperand::MONone,
       MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
 
   // Transfer chain users from old loads to the new load.
diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 7e15bdde9c112d..662439b8537778 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -545,14 +545,11 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(ptr %arg) {
 ; ptr p1.i16 = (ptr) p;
 ; (p1.i16[0] << 8) | ((i16) p[2])
 ;
-; This is essentialy a i16 load from p[1], but we don't fold the pattern now
-; because in the original DAG we don't have p[1] address available
+; This is essentially a i16 load from p[1]
 define i16 @load_i16_from_nonzero_offset(ptr %p) {
 ; CHECK-LABEL: load_i16_from_nonzero_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldrb w9, [x0, #2]
-; CHECK-NEXT:    orr w0, w9, w8, lsl #8
+; CHECK-NEXT:    ldurh w0, [x0, #1]
 ; CHECK-NEXT:    ret
   %p2.i8 = getelementptr i8, ptr %p, i64 2
   %v1 = load i16, ptr %p
@@ -569,16 +566,10 @@ define i16 @load_i16_from_nonzero_offset(ptr %p) {
 ; (p1.i16[0] << 24) | ((i32) p[2] << 16) | p3.i16[0]
 ;
 ; This is an i32 load from p[1].
-; FIXME: Should be a single load from p[1].
 define i32 @load_i32_from_nonzero_offset(ptr %p) {
 ; CHECK-LABEL: load_i32_from_nonzero_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrb w8, [x0, #2]
-; CHECK-NEXT:    ldrh w9, [x0]
-; CHECK-NEXT:    ldurh w10, [x0, #3]
-; CHECK-NEXT:    lsl w8, w8, #16
-; CHECK-NEXT:    orr w8, w8, w9, lsl #24
-; CHECK-NEXT:    orr w0, w8, w10
+; CHECK-NEXT:    ldur w0, [x0, #1]
 ; CHECK-NEXT:    ret
   %p2.i8 = getelementptr i8, ptr %p, i64 2
   %p3.i16 = getelementptr i8, ptr %p, i64 3
@@ -595,11 +586,10 @@ define i32 @load_i32_from_nonzero_offset(ptr %p) {
   ret i32 %res
 }
 
-; FIXME: Loads from %in instead of %in + 1.
 define void @load_i16_from_vector_elt1(ptr %in, ptr %out, ptr %p) {
 ; CHECK-LABEL: load_i16_from_vector_elt1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldurh w8, [x0, #1]
 ; CHECK-NEXT:    ldr w9, [x0]
 ; CHECK-NEXT:    strh w8, [x1]
 ; CHECK-NEXT:    str w9, [x2]
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index 6b6f1e62e80c48..b9af60f311562f 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -715,11 +715,10 @@ define void @short_vector_to_i64(ptr %in, ptr %out, ptr %p) {
 }
 
 ; x1 = bswap(x0[1:2])
-; FIXME: Loads from x0 instead of x0 + 1.
 define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
 ; CHECK-LABEL: short_vector_elt1_to_i16_bswap:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldurh w8, [x0, #1]
 ; CHECK-NEXT:    ldr w9, [x0]
 ; CHECK-NEXT:    rev w8, w8
 ; CHECK-NEXT:    lsr w8, w8, #16
@@ -744,16 +743,16 @@ define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
 }
 
 ; x1 = bswap(x0[1:2]), with x0 + 1 only 1-byte aligned
-; FIXME: Loads from x0 instead of x0 + 1.
 define void @short_vector_elt1_to_i16_bswap_strict_align(ptr %in, ptr %out, ptr %p) #1 {
 ; CHECK-LABEL: short_vector_elt1_to_i16_bswap_strict_align:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldr w9, [x0]
-; CHECK-NEXT:    rev w8, w8
-; CHECK-NEXT:    lsr w8, w8, #16
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #2]
+; CHECK-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-NEXT:    umov w8, v1.h[1]
+; CHECK-NEXT:    orr w8, w9, w8, lsl #8
 ; CHECK-NEXT:    strh w8, [x1]
-; CHECK-NEXT:    str w9, [x2]
+; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %p2 = getelementptr inbounds i8, ptr %in, i64 2
   %x2 = load i8, ptr %p2, align 1



More information about the llvm-branch-commits mailing list