[llvm] b9f1081 - [LoopLoadElim] Fix miscompile caused by incorrect store-to-load forwarding across mixed-width clobbers (#224833)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 01:42:33 PDT 2026
Author: Hadong Lee
Date: 2026-09-22T08:42:28Z
New Revision: b9f108134ca91564e2319ad391904b46fe5af33c
URL: https://github.com/llvm/llvm-project/commit/b9f108134ca91564e2319ad391904b46fe5af33c
DIFF: https://github.com/llvm/llvm-project/commit/b9f108134ca91564e2319ad391904b46fe5af33c.diff
LOG: [LoopLoadElim] Fix miscompile caused by incorrect store-to-load forwarding across mixed-width clobbers (#224833)
When LoopLoadElimination finds a non-forwardable Store -> Load
dependence, it currently handles it with continue and drops it entirely.
Although that store cannot provide a forwarded value itself, it may
partially clobber the value of another forwarding candidate before the
target load.
This change adds the load to LoadsWithUnsafeDependence when such a
dependence is found, preventing forwarding for that load.
The tests cover zero-distance and non-zero-distance clobbers, including
when a later full-width store overwrites the clobber.
Fixes #224832
Added:
llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
Modified:
llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index fdfbe87ad51e6..b43a04355c19b 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -190,10 +190,10 @@ class LoadEliminationForLoop {
return Candidates;
// Find store->load dependences (consequently true dep). Both lexically
- // forward and backward dependences qualify. Disqualify loads that have
- // other unknown dependences.
+ // forward and backward dependences qualify.
+ // Disqualify loads that have other unsafe dependences.
- SmallPtrSet<Instruction *, 4> LoadsWithUnknownDependence;
+ SmallPtrSet<Instruction *, 4> LoadsWithUnsafeDependence;
for (const auto &Dep : *Deps) {
Instruction *Source = Dep.getSource(DepChecker);
@@ -203,9 +203,9 @@ class LoadEliminationForLoop {
Dep.Type == MemoryDepChecker::Dependence::IndirectUnsafe ||
Dep.Type == MemoryDepChecker::Dependence::InvariantUnsafe) {
if (isa<LoadInst>(Source))
- LoadsWithUnknownDependence.insert(Source);
+ LoadsWithUnsafeDependence.insert(Source);
if (isa<LoadInst>(Destination))
- LoadsWithUnknownDependence.insert(Destination);
+ LoadsWithUnsafeDependence.insert(Destination);
continue;
}
@@ -225,17 +225,21 @@ class LoadEliminationForLoop {
continue;
// Only propagate if the stored values are bit/pointer castable.
- if (!CastInst::isBitOrNoopPointerCastable(
- getLoadStoreType(Store), getLoadStoreType(Load),
- Store->getDataLayout()))
+ if (!CastInst::isBitOrNoopPointerCastable(getLoadStoreType(Store),
+ getLoadStoreType(Load),
+ Store->getDataLayout())) {
+ // This store may partially clobber the value from another forwarding
+ // candidate.
+ LoadsWithUnsafeDependence.insert(Load);
continue;
+ }
Candidates.emplace_front(Load, Store);
}
- if (!LoadsWithUnknownDependence.empty())
+ if (!LoadsWithUnsafeDependence.empty())
Candidates.remove_if([&](const StoreToLoadForwardingCandidate &C) {
- return LoadsWithUnknownDependence.count(C.Load);
+ return LoadsWithUnsafeDependence.count(C.Load);
});
return Candidates;
diff --git a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
new file mode 100644
index 0000000000000..1939b72e41c12
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
+;
+; Do not forward a load with a non-forwardable store-to-load dependence.
+
+define void @zero_distance_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
+; CHECK-LABEL: define void @zero_distance_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: store i8 7, ptr [[P]], align 1
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[P_NEXT]], align 4
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ store i8 7, ptr %p, align 1
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i32 %z, ptr %p.next, align 4
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @nonzero_distance_clobber(ptr noalias %A, ptr noalias %Out,
+; CHECK-LABEL: define void @nonzero_distance_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
+; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+ i64 %N) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i32 42, ptr %p.next, align 4
+ store i8 7, ptr %p.next, align 1
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @overwritten_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
+; CHECK-LABEL: define void @overwritten_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
+; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i8 7, ptr %p.next, align 1
+ store i32 42, ptr %p.next, align 4
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list