[llvm-branch-commits] [llvm] [DAGCombiner] Fix MatchLoadCombine address for non-zero vector offsets (PR #244904)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Nov 21 08:16:10 PST 2026
https://github.com/lijinpei-amd created https://github.com/llvm/llvm-project/pull/244904
When the first byte has a non-zero vector element index, it is
discarded by MatchLoadCombine, which miscompiles. Also, MatchLoadCombine
bails out if the byte offset within the scalar type is non-zero.
Fix the mis-compilation by accumulating both the vector element index
and the offset within scalar elements, and adjust the load address and
alignment accordingly. Fix the missed optimization by dropping the
`MemoryByteOffset == 0` requirement.
Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>
>From a0e76e828ef92a6eead518f2fc35da79bd2a17c9 Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Sat, 21 Nov 2026 19:55:25 +0800
Subject: [PATCH 1/2] [DAGCombiner] Precommit tests for MatchLoadCombine
non-zero offsets
The byte at the lowest address comes from element 1 of the vector load,
but the combined load currently reads from the vector's base address.
In load_i32_from_nonzero_offset, it is the second byte of an i16 load,
which is currently not combined.
Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../AArch64/load-combine-big-endian.ll | 58 ++++++++++++++++++
llvm/test/CodeGen/AArch64/load-combine.ll | 59 +++++++++++++++++++
2 files changed, 117 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 3b86acc7720e39..7e15bdde9c112d 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -562,3 +562,61 @@ define i16 @load_i16_from_nonzero_offset(ptr %p) {
%res = or i16 %v1.shl, %v2
ret i16 %res
}
+
+; ptr p;
+; ptr p1.i16 = (ptr) p;
+; ptr p3.i16 = (ptr) &p[3];
+; (p1.i16[0] << 24) | ((i32) p[2] << 16) | p3.i16[0]
+;
+; This is an i32 load from p[1].
+; FIXME: Should be a single load from p[1].
+define i32 @load_i32_from_nonzero_offset(ptr %p) {
+; CHECK-LABEL: load_i32_from_nonzero_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w8, [x0, #2]
+; CHECK-NEXT: ldrh w9, [x0]
+; CHECK-NEXT: ldurh w10, [x0, #3]
+; CHECK-NEXT: lsl w8, w8, #16
+; CHECK-NEXT: orr w8, w8, w9, lsl #24
+; CHECK-NEXT: orr w0, w8, w10
+; CHECK-NEXT: ret
+ %p2.i8 = getelementptr i8, ptr %p, i64 2
+ %p3.i16 = getelementptr i8, ptr %p, i64 3
+ %v1 = load i16, ptr %p
+ %v2.i8 = load i8, ptr %p2.i8
+ %v3 = load i16, ptr %p3.i16, align 1
+ %v1.ext = zext i16 %v1 to i32
+ %v2 = zext i8 %v2.i8 to i32
+ %v3.ext = zext i16 %v3 to i32
+ %v1.shl = shl i32 %v1.ext, 24
+ %v2.shl = shl i32 %v2, 16
+ %tmp = or i32 %v1.shl, %v2.shl
+ %res = or i32 %tmp, %v3.ext
+ ret i32 %res
+}
+
+; FIXME: Loads from %in instead of %in + 1.
+define void @load_i16_from_vector_elt1(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: load_i16_from_vector_elt1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: ldr w9, [x0]
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index 0bf4fb10630258..6b6f1e62e80c48 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -714,6 +714,64 @@ define void @short_vector_to_i64(ptr %in, ptr %out, ptr %p) {
ret void
}
+; x1 = bswap(x0[1:2])
+; FIXME: Loads from x0 instead of x0 + 1.
+define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: ldr w9, [x0]
+; CHECK-NEXT: rev w8, w8
+; CHECK-NEXT: lsr w8, w8, #16
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
+
+; x1 = bswap(x0[1:2]), with x0 + 1 only 1-byte aligned
+; FIXME: Loads from x0 instead of x0 + 1.
+define void @short_vector_elt1_to_i16_bswap_strict_align(ptr %in, ptr %out, ptr %p) #1 {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap_strict_align:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: ldr w9, [x0]
+; CHECK-NEXT: rev w8, w8
+; CHECK-NEXT: lsr w8, w8, #16
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
+
; x1 = x0
define void @scalable_vector_to_i32(ptr %in, ptr %out, ptr %p) #0 {
; CHECK-LABEL: scalable_vector_to_i32:
@@ -892,3 +950,4 @@ define void @scalable_vector_to_i64(ptr %in, ptr %out, ptr %p) #0 {
}
attributes #0 = { "target-features"="+sve" }
+attributes #1 = { "target-features"="+strict-align" }
>From 1761f88599f2c9736c60dde93e446b855d71013b Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Sat, 21 Nov 2026 19:56:40 +0800
Subject: [PATCH 2/2] [DAGCombiner] Fix MatchLoadCombine address for non-zero
vector offsets
When the first byte has a non-zero vector element index, it is
discarded by MatchLoadCombine, which miscompiles. Also, MatchLoadCombine
bails out if the byte offset within the scalar type is non-zero.
Fix the mis-compilation by accumulating both the vector element index
and the offset within scalar elements, and adjust the load address and
alignment accordingly. Fix the missed optimization by dropping the
`MemoryByteOffset == 0` requirement.
Assisted-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 28 +++++++++++--------
.../AArch64/load-combine-big-endian.ll | 18 +++---------
llvm/test/CodeGen/AArch64/load-combine.ll | 15 +++++-----
3 files changed, 28 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f96fc90f7f347a..3ebbbe4846a4b8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10298,6 +10298,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
SmallPtrSet<LoadSDNode *, 8> Loads;
std::optional<SDByteProvider> FirstByteProvider;
int64_t FirstOffset = INT64_MAX;
+ uint64_t ByteOffsetInFirstLoad = 0;
// Check if all the bytes of the OR we are looking at are loaded from the same
// base address. Collect bytes offsets from Base address in ByteOffsets.
@@ -10339,11 +10340,12 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
// due to different ExtractElement indices. To accurately determine the
// byte position of an ExtractElement, we offset the base load ptr with
// the index multiplied by the byte size of each element in the vector.
+ uint64_t ByteOffsetFromVector = 0;
if (L->getMemoryVT().isVector()) {
unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
if (LoadWidthInBit % 8 != 0)
return SDValue();
- unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
+ ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
Ptr.addToOffset(ByteOffsetFromVector);
}
@@ -10361,6 +10363,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
if (ByteOffsetFromBase < FirstOffset) {
FirstByteProvider = P;
FirstOffset = ByteOffsetFromBase;
+ ByteOffsetInFirstLoad = ByteOffsetFromVector + MemoryByteOffset(*P);
}
Loads.insert(L);
@@ -10388,19 +10391,16 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
assert(FirstByteProvider && "must be set");
- // Ensure that the first byte is loaded from zero offset of the first load.
- // So the combined value can be loaded from the first load address.
- if (MemoryByteOffset(*FirstByteProvider) != 0)
- return SDValue();
auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
+ Align FirstAlign =
+ commonAlignment(FirstLoad->getAlign(), ByteOffsetInFirstLoad);
// Before legalization we allow introducing loads that are wider than legal,
// which will later be split into legally sized loads. This enables us to
// combine, for example, i8 loads forming an i64 into an i64 load, which get
// then gets split up into couple of i32 loads on 32 bit targets.
if (LegalOperations &&
- !TLI.isLoadLegal(VT, MemVT, FirstLoad->getAlign(),
- FirstLoad->getAddressSpace(),
+ !TLI.isLoadLegal(VT, MemVT, FirstAlign, FirstLoad->getAddressSpace(),
NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, false))
return SDValue();
@@ -10430,16 +10430,22 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
unsigned Fast = 0;
bool Allowed =
TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
- *FirstLoad->getMemOperand(), &Fast);
+ FirstLoad->getAddressSpace(), FirstAlign,
+ FirstLoad->getMemOperand()->getFlags(), &Fast);
if (!Allowed || !Fast)
return SDValue();
const MDNode *MemCacheHint = getCommonMemCacheHint(Loads);
+ SDValue Ptr = FirstLoad->getBasePtr();
+ if (ByteOffsetInFirstLoad)
+ Ptr = DAG.getMemBasePlusOffset(
+ Ptr, TypeSize::getFixed(ByteOffsetInFirstLoad), SDLoc(N));
+
SDValue NewLoad = DAG.getExtLoad(
- NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
- FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), MemVT,
- FirstLoad->getAlign(), MachineMemOperand::MONone,
+ NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain, Ptr,
+ FirstLoad->getPointerInfo().getWithOffset(ByteOffsetInFirstLoad), MemVT,
+ FirstAlign, MachineMemOperand::MONone,
MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
// Transfer chain users from old loads to the new load.
diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 7e15bdde9c112d..662439b8537778 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -545,14 +545,11 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(ptr %arg) {
; ptr p1.i16 = (ptr) p;
; (p1.i16[0] << 8) | ((i16) p[2])
;
-; This is essentialy a i16 load from p[1], but we don't fold the pattern now
-; because in the original DAG we don't have p[1] address available
+; This is essentially a i16 load from p[1]
define i16 @load_i16_from_nonzero_offset(ptr %p) {
; CHECK-LABEL: load_i16_from_nonzero_offset:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: ldrb w9, [x0, #2]
-; CHECK-NEXT: orr w0, w9, w8, lsl #8
+; CHECK-NEXT: ldurh w0, [x0, #1]
; CHECK-NEXT: ret
%p2.i8 = getelementptr i8, ptr %p, i64 2
%v1 = load i16, ptr %p
@@ -569,16 +566,10 @@ define i16 @load_i16_from_nonzero_offset(ptr %p) {
; (p1.i16[0] << 24) | ((i32) p[2] << 16) | p3.i16[0]
;
; This is an i32 load from p[1].
-; FIXME: Should be a single load from p[1].
define i32 @load_i32_from_nonzero_offset(ptr %p) {
; CHECK-LABEL: load_i32_from_nonzero_offset:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrb w8, [x0, #2]
-; CHECK-NEXT: ldrh w9, [x0]
-; CHECK-NEXT: ldurh w10, [x0, #3]
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: orr w8, w8, w9, lsl #24
-; CHECK-NEXT: orr w0, w8, w10
+; CHECK-NEXT: ldur w0, [x0, #1]
; CHECK-NEXT: ret
%p2.i8 = getelementptr i8, ptr %p, i64 2
%p3.i16 = getelementptr i8, ptr %p, i64 3
@@ -595,11 +586,10 @@ define i32 @load_i32_from_nonzero_offset(ptr %p) {
ret i32 %res
}
-; FIXME: Loads from %in instead of %in + 1.
define void @load_i16_from_vector_elt1(ptr %in, ptr %out, ptr %p) {
; CHECK-LABEL: load_i16_from_vector_elt1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: ldurh w8, [x0, #1]
; CHECK-NEXT: ldr w9, [x0]
; CHECK-NEXT: strh w8, [x1]
; CHECK-NEXT: str w9, [x2]
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index 6b6f1e62e80c48..b9af60f311562f 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -715,11 +715,10 @@ define void @short_vector_to_i64(ptr %in, ptr %out, ptr %p) {
}
; x1 = bswap(x0[1:2])
-; FIXME: Loads from x0 instead of x0 + 1.
define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
; CHECK-LABEL: short_vector_elt1_to_i16_bswap:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: ldurh w8, [x0, #1]
; CHECK-NEXT: ldr w9, [x0]
; CHECK-NEXT: rev w8, w8
; CHECK-NEXT: lsr w8, w8, #16
@@ -744,16 +743,16 @@ define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
}
; x1 = bswap(x0[1:2]), with x0 + 1 only 1-byte aligned
-; FIXME: Loads from x0 instead of x0 + 1.
define void @short_vector_elt1_to_i16_bswap_strict_align(ptr %in, ptr %out, ptr %p) #1 {
; CHECK-LABEL: short_vector_elt1_to_i16_bswap_strict_align:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: ldr w9, [x0]
-; CHECK-NEXT: rev w8, w8
-; CHECK-NEXT: lsr w8, w8, #16
+; CHECK-NEXT: ldr s0, [x0]
+; CHECK-NEXT: ldrb w9, [x0, #2]
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: umov w8, v1.h[1]
+; CHECK-NEXT: orr w8, w9, w8, lsl #8
; CHECK-NEXT: strh w8, [x1]
-; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: str s0, [x2]
; CHECK-NEXT: ret
%p2 = getelementptr inbounds i8, ptr %in, i64 2
%x2 = load i8, ptr %p2, align 1
More information about the llvm-branch-commits
mailing list