[llvm-branch-commits] [llvm] [DAGCombiner] Fix MatchLoadCombine address for non-zero vector offsets (PR #244904)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Nov 21 08:16:42 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: lijinpei-amd
<details>
<summary>Changes</summary>
When the first byte has a non-zero vector element index, it is
discarded by MatchLoadCombine, which miscompiles. Also, MatchLoadCombine
bails out if the byte offset within the scalar type is non-zero.
Fix the mis-compilation by accumulating both the vector element index
and the offset within scalar elements, and adjust the load address and
alignment accordingly. Fix the missed optimization by dropping the
`MemoryByteOffset == 0` requirement.
Assisted-by: Claude Opus 5.5 <noreply@<!-- -->anthropic.com>
---
Full diff: https://github.com/llvm/llvm-project/pull/244904.diff
3 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+17-11)
- (modified) llvm/test/CodeGen/AArch64/load-combine-big-endian.ll (+53-5)
- (modified) llvm/test/CodeGen/AArch64/load-combine.ll (+58)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f96fc90f7f347..3ebbbe4846a4b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10298,6 +10298,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
SmallPtrSet<LoadSDNode *, 8> Loads;
std::optional<SDByteProvider> FirstByteProvider;
int64_t FirstOffset = INT64_MAX;
+ uint64_t ByteOffsetInFirstLoad = 0;
// Check if all the bytes of the OR we are looking at are loaded from the same
// base address. Collect bytes offsets from Base address in ByteOffsets.
@@ -10339,11 +10340,12 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
// due to different ExtractElement indices. To accurately determine the
// byte position of an ExtractElement, we offset the base load ptr with
// the index multiplied by the byte size of each element in the vector.
+ uint64_t ByteOffsetFromVector = 0;
if (L->getMemoryVT().isVector()) {
unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
if (LoadWidthInBit % 8 != 0)
return SDValue();
- unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
+ ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
Ptr.addToOffset(ByteOffsetFromVector);
}
@@ -10361,6 +10363,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
if (ByteOffsetFromBase < FirstOffset) {
FirstByteProvider = P;
FirstOffset = ByteOffsetFromBase;
+ ByteOffsetInFirstLoad = ByteOffsetFromVector + MemoryByteOffset(*P);
}
Loads.insert(L);
@@ -10388,19 +10391,16 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
assert(FirstByteProvider && "must be set");
- // Ensure that the first byte is loaded from zero offset of the first load.
- // So the combined value can be loaded from the first load address.
- if (MemoryByteOffset(*FirstByteProvider) != 0)
- return SDValue();
auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
+ Align FirstAlign =
+ commonAlignment(FirstLoad->getAlign(), ByteOffsetInFirstLoad);
// Before legalization we allow introducing loads that are wider than legal,
// which will later be split into legally sized loads. This enables us to
// combine, for example, i8 loads forming an i64 into an i64 load, which get
// then gets split up into couple of i32 loads on 32 bit targets.
if (LegalOperations &&
- !TLI.isLoadLegal(VT, MemVT, FirstLoad->getAlign(),
- FirstLoad->getAddressSpace(),
+ !TLI.isLoadLegal(VT, MemVT, FirstAlign, FirstLoad->getAddressSpace(),
NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, false))
return SDValue();
@@ -10430,16 +10430,22 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
unsigned Fast = 0;
bool Allowed =
TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
- *FirstLoad->getMemOperand(), &Fast);
+ FirstLoad->getAddressSpace(), FirstAlign,
+ FirstLoad->getMemOperand()->getFlags(), &Fast);
if (!Allowed || !Fast)
return SDValue();
const MDNode *MemCacheHint = getCommonMemCacheHint(Loads);
+ SDValue Ptr = FirstLoad->getBasePtr();
+ if (ByteOffsetInFirstLoad)
+ Ptr = DAG.getMemBasePlusOffset(
+ Ptr, TypeSize::getFixed(ByteOffsetInFirstLoad), SDLoc(N));
+
SDValue NewLoad = DAG.getExtLoad(
- NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
- FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), MemVT,
- FirstLoad->getAlign(), MachineMemOperand::MONone,
+ NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain, Ptr,
+ FirstLoad->getPointerInfo().getWithOffset(ByteOffsetInFirstLoad), MemVT,
+ FirstAlign, MachineMemOperand::MONone,
MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
// Transfer chain users from old loads to the new load.
diff --git a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
index 3b86acc7720e3..662439b853777 100644
--- a/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine-big-endian.ll
@@ -545,14 +545,11 @@ define i32 @zext_load_i32_by_i8_bswap_shl_16(ptr %arg) {
; ptr p1.i16 = (ptr) p;
; (p1.i16[0] << 8) | ((i16) p[2])
;
-; This is essentialy a i16 load from p[1], but we don't fold the pattern now
-; because in the original DAG we don't have p[1] address available
+; This is essentially a i16 load from p[1]
define i16 @load_i16_from_nonzero_offset(ptr %p) {
; CHECK-LABEL: load_i16_from_nonzero_offset:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: ldrb w9, [x0, #2]
-; CHECK-NEXT: orr w0, w9, w8, lsl #8
+; CHECK-NEXT: ldurh w0, [x0, #1]
; CHECK-NEXT: ret
%p2.i8 = getelementptr i8, ptr %p, i64 2
%v1 = load i16, ptr %p
@@ -562,3 +559,54 @@ define i16 @load_i16_from_nonzero_offset(ptr %p) {
%res = or i16 %v1.shl, %v2
ret i16 %res
}
+
+; ptr p;
+; ptr p1.i16 = (ptr) p;
+; ptr p3.i16 = (ptr) &p[3];
+; (p1.i16[0] << 24) | ((i32) p[2] << 16) | p3.i16[0]
+;
+; This is an i32 load from p[1].
+define i32 @load_i32_from_nonzero_offset(ptr %p) {
+; CHECK-LABEL: load_i32_from_nonzero_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldur w0, [x0, #1]
+; CHECK-NEXT: ret
+ %p2.i8 = getelementptr i8, ptr %p, i64 2
+ %p3.i16 = getelementptr i8, ptr %p, i64 3
+ %v1 = load i16, ptr %p
+ %v2.i8 = load i8, ptr %p2.i8
+ %v3 = load i16, ptr %p3.i16, align 1
+ %v1.ext = zext i16 %v1 to i32
+ %v2 = zext i8 %v2.i8 to i32
+ %v3.ext = zext i16 %v3 to i32
+ %v1.shl = shl i32 %v1.ext, 24
+ %v2.shl = shl i32 %v2, 16
+ %tmp = or i32 %v1.shl, %v2.shl
+ %res = or i32 %tmp, %v3.ext
+ ret i32 %res
+}
+
+define void @load_i16_from_vector_elt1(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: load_i16_from_vector_elt1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldurh w8, [x0, #1]
+; CHECK-NEXT: ldr w9, [x0]
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/load-combine.ll b/llvm/test/CodeGen/AArch64/load-combine.ll
index 0bf4fb1063025..b9af60f311562 100644
--- a/llvm/test/CodeGen/AArch64/load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/load-combine.ll
@@ -714,6 +714,63 @@ define void @short_vector_to_i64(ptr %in, ptr %out, ptr %p) {
ret void
}
+; x1 = bswap(x0[1:2])
+define void @short_vector_elt1_to_i16_bswap(ptr %in, ptr %out, ptr %p) {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldurh w8, [x0, #1]
+; CHECK-NEXT: ldr w9, [x0]
+; CHECK-NEXT: rev w8, w8
+; CHECK-NEXT: lsr w8, w8, #16
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
+
+; x1 = bswap(x0[1:2]), with x0 + 1 only 1-byte aligned
+define void @short_vector_elt1_to_i16_bswap_strict_align(ptr %in, ptr %out, ptr %p) #1 {
+; CHECK-LABEL: short_vector_elt1_to_i16_bswap_strict_align:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr s0, [x0]
+; CHECK-NEXT: ldrb w9, [x0, #2]
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: umov w8, v1.h[1]
+; CHECK-NEXT: orr w8, w9, w8, lsl #8
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: str s0, [x2]
+; CHECK-NEXT: ret
+ %p2 = getelementptr inbounds i8, ptr %in, i64 2
+ %x2 = load i8, ptr %p2, align 1
+ %ld = load <4 x i8>, ptr %in, align 4
+
+ %e1 = extractelement <4 x i8> %ld, i32 1
+
+ %z0 = zext i8 %x2 to i16
+ %z1 = zext i8 %e1 to i16
+
+ %s1 = shl nuw i16 %z1, 8
+ %i = or i16 %s1, %z0
+
+ store i16 %i, ptr %out
+ store <4 x i8> %ld, ptr %p, align 4
+ ret void
+}
+
; x1 = x0
define void @scalable_vector_to_i32(ptr %in, ptr %out, ptr %p) #0 {
; CHECK-LABEL: scalable_vector_to_i32:
@@ -892,3 +949,4 @@ define void @scalable_vector_to_i64(ptr %in, ptr %out, ptr %p) #0 {
}
attributes #0 = { "target-features"="+sve" }
+attributes #1 = { "target-features"="+strict-align" }
``````````
</details>
https://github.com/llvm/llvm-project/pull/244904
More information about the llvm-branch-commits
mailing list