[llvm] [LoopLoadElim] Fix miscompile caused by incorrect store-to-load forwarding across mixed-width clobbers (PR #224833)
Hadong Lee via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 02:23:39 PDT 2026
https://github.com/ChrisLee02 updated https://github.com/llvm/llvm-project/pull/224833
>From 9e2415e5469aec3ec2f157afb2e91bfc1c20a9ea Mon Sep 17 00:00:00 2001
From: Chrislee02 <hdchris765 at snu.ac.kr>
Date: Sat, 19 Sep 2026 22:35:41 +0900
Subject: [PATCH 1/4] [LoopLoadElim] Add tests for incorrect store-to-load
forwarding across mixed-width clobbers
---
.../LoopLoadElim/mixed-width-clobber.ll | 138 ++++++++++++++++++
1 file changed, 138 insertions(+)
create mode 100644 llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
diff --git a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
new file mode 100644
index 0000000000000..e2b7f93cf2718
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
+;
+; Do not forward across a partial clobber, but preserve forwarding when a
+; later full-width store overwrites the clobber.
+
+define void @zero_distance_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
+; CHECK-LABEL: define void @zero_distance_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ [[Z:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: store i8 7, ptr [[P]], align 1
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[P_NEXT]], align 4
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ store i8 7, ptr %p, align 1
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i32 %z, ptr %p.next, align 4
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @nonzero_distance_clobber(ptr noalias %A, ptr noalias %Out,
+; CHECK-LABEL: define void @nonzero_distance_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ 42, %[[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
+; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+ i64 %N) {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i32 42, ptr %p.next, align 4
+ store i8 7, ptr %p.next, align 1
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @overwritten_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
+; CHECK-LABEL: define void @overwritten_clobber(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ 42, %[[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
+; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
+; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %A, i64 %i
+ %i.next = add nuw nsw i64 %i, 1
+ %p.next = getelementptr inbounds i32, ptr %A, i64 %i.next
+ store i8 7, ptr %p.next, align 1
+ store i32 42, ptr %p.next, align 4
+ %x = load i32, ptr %p, align 4
+ %z = add i32 %x, 1
+ %result = getelementptr inbounds i32, ptr %Out, i64 %i
+ store i32 %z, ptr %result, align 4
+ %c = icmp ult i64 %i.next, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
>From 30201d92be81e4c5435dd1b2ab92c68eae52e48d Mon Sep 17 00:00:00 2001
From: Chrislee02 <hdchris765 at snu.ac.kr>
Date: Sat, 19 Sep 2026 23:47:01 +0900
Subject: [PATCH 2/4] [LoopLoadElim] Refactor store-to-load dependence
collection for potential clobbers
---
.../Transforms/Scalar/LoopLoadElimination.cpp | 56 +++++++++++++------
1 file changed, 40 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index fdfbe87ad51e6..04eebb29f05f9 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -147,6 +147,11 @@ struct StoreToLoadForwardingCandidate {
#endif
};
+struct StoreToLoadDependences {
+ std::forward_list<StoreToLoadForwardingCandidate> Candidates;
+ std::forward_list<StoreToLoadForwardingCandidate> PotentialClobbers;
+};
+
} // end anonymous namespace
/// Check if the store dominates all latches, so as long as there is no
@@ -180,14 +185,13 @@ class LoadEliminationForLoop {
///
/// Note that no candidate is returned if LAA has failed to analyze the loop
/// (e.g. if it's not bottom-tested, contains volatile memops, etc.)
- std::forward_list<StoreToLoadForwardingCandidate>
- findStoreToLoadDependences(const LoopAccessInfo &LAI) {
- std::forward_list<StoreToLoadForwardingCandidate> Candidates;
+ StoreToLoadDependences findStoreToLoadDependences(const LoopAccessInfo &LAI) {
+ StoreToLoadDependences Dependences;
const auto &DepChecker = LAI.getDepChecker();
const auto *Deps = DepChecker.getDependences();
if (!Deps)
- return Candidates;
+ return Dependences;
// Find store->load dependences (consequently true dep). Both lexically
// forward and backward dependences qualify. Disqualify loads that have
@@ -225,20 +229,25 @@ class LoadEliminationForLoop {
continue;
// Only propagate if the stored values are bit/pointer castable.
- if (!CastInst::isBitOrNoopPointerCastable(
- getLoadStoreType(Store), getLoadStoreType(Load),
- Store->getDataLayout()))
+ // if the types are not bitcastable, add the candidate to the potential
+ // clobbers list
+ if (!CastInst::isBitOrNoopPointerCastable(getLoadStoreType(Store),
+ getLoadStoreType(Load),
+ Store->getDataLayout())) {
+ Dependences.PotentialClobbers.emplace_front(Load, Store);
continue;
+ }
- Candidates.emplace_front(Load, Store);
+ Dependences.Candidates.emplace_front(Load, Store);
}
if (!LoadsWithUnknownDependence.empty())
- Candidates.remove_if([&](const StoreToLoadForwardingCandidate &C) {
- return LoadsWithUnknownDependence.count(C.Load);
- });
+ Dependences.Candidates.remove_if(
+ [&](const StoreToLoadForwardingCandidate &C) {
+ return LoadsWithUnknownDependence.count(C.Load);
+ });
- return Candidates;
+ return Dependences;
}
/// Return the index of the instruction according to program order.
@@ -248,6 +257,15 @@ class LoadEliminationForLoop {
return I->second;
}
+ /// Remove candidates whose forwarded value is clobbered before the load in
+ /// the next iteration.
+ void removeCandidatesWithPotentialClobbers(
+ SmallVectorImpl<StoreToLoadForwardingCandidate> &Candidates,
+ const std::forward_list<StoreToLoadForwardingCandidate>
+ &PotentialClobbers) {
+ // TODO: Use potential clobbers to filter forwarding candidates.
+ }
+
/// If a load has multiple candidates associated (i.e. different
/// stores), it means that it could be forwarding from multiple stores
/// depending on control flow. Remove these candidates.
@@ -511,7 +529,7 @@ class LoadEliminationForLoop {
// First start with store->load dependences.
auto StoreToLoadDependences = findStoreToLoadDependences(LAI);
- if (StoreToLoadDependences.empty())
+ if (StoreToLoadDependences.Candidates.empty())
return false;
// Generate an index for each load and store according to the original
@@ -520,13 +538,14 @@ class LoadEliminationForLoop {
// To keep things simple for now, remove those where the load is potentially
// fed by multiple stores.
- removeDependencesFromMultipleStores(StoreToLoadDependences);
- if (StoreToLoadDependences.empty())
+ removeDependencesFromMultipleStores(StoreToLoadDependences.Candidates);
+ if (StoreToLoadDependences.Candidates.empty())
return false;
// Filter the candidates further.
SmallVector<StoreToLoadForwardingCandidate, 4> Candidates;
- for (const StoreToLoadForwardingCandidate &Cand : StoreToLoadDependences) {
+ for (const StoreToLoadForwardingCandidate &Cand :
+ StoreToLoadDependences.Candidates) {
LLVM_DEBUG(dbgs() << "Candidate " << Cand);
// Make sure that the stored values is available everywhere in the loop in
@@ -560,6 +579,11 @@ class LoadEliminationForLoop {
if (Candidates.empty())
return false;
+ removeCandidatesWithPotentialClobbers(
+ Candidates, StoreToLoadDependences.PotentialClobbers);
+ if (Candidates.empty())
+ return false;
+
// Check intervening may-alias stores. These need runtime checks for alias
// disambiguation.
SmallVector<RuntimePointerCheck, 4> Checks = collectMemchecks(Candidates);
>From 98fd7f22686eb916b9f61ebc68d58a3ccc9bf778 Mon Sep 17 00:00:00 2001
From: Chrislee02 <hdchris765 at snu.ac.kr>
Date: Sun, 20 Sep 2026 00:46:07 +0900
Subject: [PATCH 3/4] [LoopLoadElim] Filter candidates clobbered by mixed-width
stores
---
.../Transforms/Scalar/LoopLoadElimination.cpp | 83 ++++++++++++++++++-
.../LoopLoadElim/mixed-width-clobber.ll | 8 +-
2 files changed, 84 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 04eebb29f05f9..1f9a51cee5d11 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -57,6 +57,7 @@
#include <algorithm>
#include <cassert>
#include <forward_list>
+#include <optional>
#include <tuple>
#include <utility>
@@ -263,7 +264,87 @@ class LoadEliminationForLoop {
SmallVectorImpl<StoreToLoadForwardingCandidate> &Candidates,
const std::forward_list<StoreToLoadForwardingCandidate>
&PotentialClobbers) {
- // TODO: Use potential clobbers to filter forwarding candidates.
+ auto GetStoreToLoadDistance =
+ [&](const StoreToLoadForwardingCandidate &Clobber)
+ -> std::optional<int64_t> {
+ Value *LoadPtr = Clobber.Load->getPointerOperand();
+ Value *StorePtr = Clobber.Store->getPointerOperand();
+ Type *LoadType = getLoadStoreType(Clobber.Load);
+ const DataLayout &DL = Clobber.Load->getDataLayout();
+
+ if (LoadPtr->getType()->getPointerAddressSpace() !=
+ StorePtr->getType()->getPointerAddressSpace())
+ return std::nullopt;
+
+ int64_t StrideLoad =
+ getPtrStride(PSE, LoadType, LoadPtr, L, *DT).value_or(0);
+ int64_t StrideStore =
+ getPtrStride(PSE, LoadType, StorePtr, L, *DT).value_or(0);
+ if (!StrideLoad || StrideLoad != StrideStore)
+ return std::nullopt;
+
+ auto *LoadPtrSCEV = dyn_cast<SCEVAddRecExpr>(PSE.getSCEV(LoadPtr));
+ auto *StorePtrSCEV = dyn_cast<SCEVAddRecExpr>(PSE.getSCEV(StorePtr));
+ if (!LoadPtrSCEV || !StorePtrSCEV)
+ return std::nullopt;
+
+ auto *ByteDistance = dyn_cast<SCEVConstant>(
+ PSE.getSE()->getMinusSCEV(StorePtrSCEV, LoadPtrSCEV));
+ if (!ByteDistance || !ByteDistance->getAPInt().isSignedIntN(64))
+ return std::nullopt;
+
+ int64_t StrideInBytes =
+ static_cast<int64_t>(DL.getTypeAllocSize(LoadType)) * StrideLoad;
+ if (!StrideInBytes ||
+ ByteDistance->getAPInt().getSExtValue() % StrideInBytes != 0)
+ return std::nullopt;
+
+ return ByteDistance->getAPInt().getSExtValue() / StrideInBytes;
+ };
+
+ auto ClobbersForwardedValue =
+ [&](const StoreToLoadForwardingCandidate &Candidate,
+ const StoreToLoadForwardingCandidate &Clobber) {
+ std::optional<int64_t> Distance = GetStoreToLoadDistance(Clobber);
+ // If we can't determine the distance, we can't prove that the clobber
+ // doesn't overwrite the candidate's value.
+ if (!Distance)
+ return true;
+
+ // findStoreToLoadDependences normalizes backward dependencies to
+ // store-to-load order, so the distance cannot be negative.
+ if (*Distance < 0)
+ llvm_unreachable(
+ "Store-to-load dependence must not have negative distance");
+
+ // The candidate's dependence distance is 1.
+ //
+ // A distance-0 clobber occurs in the load's iteration.
+ // Candidate -> Clobber -> Load.
+ if (*Distance == 0)
+ return true;
+
+ // A clobber with distance greater than 1 occurs before the candidate
+ // store, which overwrites it. Clobber -> Candidate -> Load.
+ if (*Distance > 1)
+ return false;
+
+ // With distance 1, both stores occur in the candidate's source
+ // iteration. Candidate -> Clobber -> Load if Candidate.Store precedes
+ // Clobber.Store; otherwise, Clobber -> Candidate -> Load.
+ return getInstrIndex(Candidate.Store) < getInstrIndex(Clobber.Store);
+ };
+
+ llvm::erase_if(Candidates,
+ [&](const StoreToLoadForwardingCandidate &Candidate) {
+ for (const auto &Clobber : PotentialClobbers) {
+ if (Clobber.Load != Candidate.Load)
+ continue;
+ if (ClobbersForwardedValue(Candidate, Clobber))
+ return true;
+ }
+ return false;
+ });
}
/// If a load has multiple candidates associated (i.e. different
diff --git a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
index e2b7f93cf2718..1c26c90b7ea8a 100644
--- a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
+++ b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
@@ -8,15 +8,13 @@ define void @zero_distance_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
; CHECK-LABEL: define void @zero_distance_clobber(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ [[Z:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: store i8 7, ptr [[P]], align 1
; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[Z]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_NEXT]]
; CHECK-NEXT: store i32 [[Z]], ptr [[P_NEXT]], align 4
@@ -52,10 +50,8 @@ define void @nonzero_distance_clobber(ptr noalias %A, ptr noalias %Out,
; CHECK-LABEL: define void @nonzero_distance_clobber(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ 42, %[[LOOP]] ]
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -63,7 +59,7 @@ define void @nonzero_distance_clobber(ptr noalias %A, ptr noalias %Out,
; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[Z:%.*]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
>From daefafd5d8849321b2f09c5ab81dbedcda798152 Mon Sep 17 00:00:00 2001
From: Chrislee02 <hdchris765 at snu.ac.kr>
Date: Sun, 20 Sep 2026 20:09:46 +0900
Subject: [PATCH 4/4] [LoopLoadElim] Prevent forwarding across mixed-width
stores
---
.../Transforms/Scalar/LoopLoadElimination.cpp | 145 +++---------------
.../LoopLoadElim/mixed-width-clobber.ll | 7 +-
2 files changed, 24 insertions(+), 128 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 1f9a51cee5d11..b43a04355c19b 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -57,7 +57,6 @@
#include <algorithm>
#include <cassert>
#include <forward_list>
-#include <optional>
#include <tuple>
#include <utility>
@@ -148,11 +147,6 @@ struct StoreToLoadForwardingCandidate {
#endif
};
-struct StoreToLoadDependences {
- std::forward_list<StoreToLoadForwardingCandidate> Candidates;
- std::forward_list<StoreToLoadForwardingCandidate> PotentialClobbers;
-};
-
} // end anonymous namespace
/// Check if the store dominates all latches, so as long as there is no
@@ -186,19 +180,20 @@ class LoadEliminationForLoop {
///
/// Note that no candidate is returned if LAA has failed to analyze the loop
/// (e.g. if it's not bottom-tested, contains volatile memops, etc.)
- StoreToLoadDependences findStoreToLoadDependences(const LoopAccessInfo &LAI) {
- StoreToLoadDependences Dependences;
+ std::forward_list<StoreToLoadForwardingCandidate>
+ findStoreToLoadDependences(const LoopAccessInfo &LAI) {
+ std::forward_list<StoreToLoadForwardingCandidate> Candidates;
const auto &DepChecker = LAI.getDepChecker();
const auto *Deps = DepChecker.getDependences();
if (!Deps)
- return Dependences;
+ return Candidates;
// Find store->load dependences (consequently true dep). Both lexically
- // forward and backward dependences qualify. Disqualify loads that have
- // other unknown dependences.
+ // forward and backward dependences qualify.
+ // Disqualify loads that have other unsafe dependences.
- SmallPtrSet<Instruction *, 4> LoadsWithUnknownDependence;
+ SmallPtrSet<Instruction *, 4> LoadsWithUnsafeDependence;
for (const auto &Dep : *Deps) {
Instruction *Source = Dep.getSource(DepChecker);
@@ -208,9 +203,9 @@ class LoadEliminationForLoop {
Dep.Type == MemoryDepChecker::Dependence::IndirectUnsafe ||
Dep.Type == MemoryDepChecker::Dependence::InvariantUnsafe) {
if (isa<LoadInst>(Source))
- LoadsWithUnknownDependence.insert(Source);
+ LoadsWithUnsafeDependence.insert(Source);
if (isa<LoadInst>(Destination))
- LoadsWithUnknownDependence.insert(Destination);
+ LoadsWithUnsafeDependence.insert(Destination);
continue;
}
@@ -230,25 +225,24 @@ class LoadEliminationForLoop {
continue;
// Only propagate if the stored values are bit/pointer castable.
- // if the types are not bitcastable, add the candidate to the potential
- // clobbers list
if (!CastInst::isBitOrNoopPointerCastable(getLoadStoreType(Store),
getLoadStoreType(Load),
Store->getDataLayout())) {
- Dependences.PotentialClobbers.emplace_front(Load, Store);
+ // This store may partially clobber the value from another forwarding
+ // candidate.
+ LoadsWithUnsafeDependence.insert(Load);
continue;
}
- Dependences.Candidates.emplace_front(Load, Store);
+ Candidates.emplace_front(Load, Store);
}
- if (!LoadsWithUnknownDependence.empty())
- Dependences.Candidates.remove_if(
- [&](const StoreToLoadForwardingCandidate &C) {
- return LoadsWithUnknownDependence.count(C.Load);
- });
+ if (!LoadsWithUnsafeDependence.empty())
+ Candidates.remove_if([&](const StoreToLoadForwardingCandidate &C) {
+ return LoadsWithUnsafeDependence.count(C.Load);
+ });
- return Dependences;
+ return Candidates;
}
/// Return the index of the instruction according to program order.
@@ -258,95 +252,6 @@ class LoadEliminationForLoop {
return I->second;
}
- /// Remove candidates whose forwarded value is clobbered before the load in
- /// the next iteration.
- void removeCandidatesWithPotentialClobbers(
- SmallVectorImpl<StoreToLoadForwardingCandidate> &Candidates,
- const std::forward_list<StoreToLoadForwardingCandidate>
- &PotentialClobbers) {
- auto GetStoreToLoadDistance =
- [&](const StoreToLoadForwardingCandidate &Clobber)
- -> std::optional<int64_t> {
- Value *LoadPtr = Clobber.Load->getPointerOperand();
- Value *StorePtr = Clobber.Store->getPointerOperand();
- Type *LoadType = getLoadStoreType(Clobber.Load);
- const DataLayout &DL = Clobber.Load->getDataLayout();
-
- if (LoadPtr->getType()->getPointerAddressSpace() !=
- StorePtr->getType()->getPointerAddressSpace())
- return std::nullopt;
-
- int64_t StrideLoad =
- getPtrStride(PSE, LoadType, LoadPtr, L, *DT).value_or(0);
- int64_t StrideStore =
- getPtrStride(PSE, LoadType, StorePtr, L, *DT).value_or(0);
- if (!StrideLoad || StrideLoad != StrideStore)
- return std::nullopt;
-
- auto *LoadPtrSCEV = dyn_cast<SCEVAddRecExpr>(PSE.getSCEV(LoadPtr));
- auto *StorePtrSCEV = dyn_cast<SCEVAddRecExpr>(PSE.getSCEV(StorePtr));
- if (!LoadPtrSCEV || !StorePtrSCEV)
- return std::nullopt;
-
- auto *ByteDistance = dyn_cast<SCEVConstant>(
- PSE.getSE()->getMinusSCEV(StorePtrSCEV, LoadPtrSCEV));
- if (!ByteDistance || !ByteDistance->getAPInt().isSignedIntN(64))
- return std::nullopt;
-
- int64_t StrideInBytes =
- static_cast<int64_t>(DL.getTypeAllocSize(LoadType)) * StrideLoad;
- if (!StrideInBytes ||
- ByteDistance->getAPInt().getSExtValue() % StrideInBytes != 0)
- return std::nullopt;
-
- return ByteDistance->getAPInt().getSExtValue() / StrideInBytes;
- };
-
- auto ClobbersForwardedValue =
- [&](const StoreToLoadForwardingCandidate &Candidate,
- const StoreToLoadForwardingCandidate &Clobber) {
- std::optional<int64_t> Distance = GetStoreToLoadDistance(Clobber);
- // If we can't determine the distance, we can't prove that the clobber
- // doesn't overwrite the candidate's value.
- if (!Distance)
- return true;
-
- // findStoreToLoadDependences normalizes backward dependencies to
- // store-to-load order, so the distance cannot be negative.
- if (*Distance < 0)
- llvm_unreachable(
- "Store-to-load dependence must not have negative distance");
-
- // The candidate's dependence distance is 1.
- //
- // A distance-0 clobber occurs in the load's iteration.
- // Candidate -> Clobber -> Load.
- if (*Distance == 0)
- return true;
-
- // A clobber with distance greater than 1 occurs before the candidate
- // store, which overwrites it. Clobber -> Candidate -> Load.
- if (*Distance > 1)
- return false;
-
- // With distance 1, both stores occur in the candidate's source
- // iteration. Candidate -> Clobber -> Load if Candidate.Store precedes
- // Clobber.Store; otherwise, Clobber -> Candidate -> Load.
- return getInstrIndex(Candidate.Store) < getInstrIndex(Clobber.Store);
- };
-
- llvm::erase_if(Candidates,
- [&](const StoreToLoadForwardingCandidate &Candidate) {
- for (const auto &Clobber : PotentialClobbers) {
- if (Clobber.Load != Candidate.Load)
- continue;
- if (ClobbersForwardedValue(Candidate, Clobber))
- return true;
- }
- return false;
- });
- }
-
/// If a load has multiple candidates associated (i.e. different
/// stores), it means that it could be forwarding from multiple stores
/// depending on control flow. Remove these candidates.
@@ -610,7 +515,7 @@ class LoadEliminationForLoop {
// First start with store->load dependences.
auto StoreToLoadDependences = findStoreToLoadDependences(LAI);
- if (StoreToLoadDependences.Candidates.empty())
+ if (StoreToLoadDependences.empty())
return false;
// Generate an index for each load and store according to the original
@@ -619,14 +524,13 @@ class LoadEliminationForLoop {
// To keep things simple for now, remove those where the load is potentially
// fed by multiple stores.
- removeDependencesFromMultipleStores(StoreToLoadDependences.Candidates);
- if (StoreToLoadDependences.Candidates.empty())
+ removeDependencesFromMultipleStores(StoreToLoadDependences);
+ if (StoreToLoadDependences.empty())
return false;
// Filter the candidates further.
SmallVector<StoreToLoadForwardingCandidate, 4> Candidates;
- for (const StoreToLoadForwardingCandidate &Cand :
- StoreToLoadDependences.Candidates) {
+ for (const StoreToLoadForwardingCandidate &Cand : StoreToLoadDependences) {
LLVM_DEBUG(dbgs() << "Candidate " << Cand);
// Make sure that the stored values is available everywhere in the loop in
@@ -660,11 +564,6 @@ class LoadEliminationForLoop {
if (Candidates.empty())
return false;
- removeCandidatesWithPotentialClobbers(
- Candidates, StoreToLoadDependences.PotentialClobbers);
- if (Candidates.empty())
- return false;
-
// Check intervening may-alias stores. These need runtime checks for alias
// disambiguation.
SmallVector<RuntimePointerCheck, 4> Checks = collectMemchecks(Candidates);
diff --git a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
index 1c26c90b7ea8a..1939b72e41c12 100644
--- a/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
+++ b/llvm/test/Transforms/LoopLoadElim/mixed-width-clobber.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
;
-; Do not forward across a partial clobber, but preserve forwarding when a
-; later full-width store overwrites the clobber.
+; Do not forward a load with a non-forwardable store-to-load dependence.
define void @zero_distance_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
; CHECK-LABEL: define void @zero_distance_clobber(
@@ -93,10 +92,8 @@ define void @overwritten_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
; CHECK-LABEL: define void @overwritten_clobber(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], %[[ENTRY]] ], [ 42, %[[LOOP]] ]
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
@@ -104,7 +101,7 @@ define void @overwritten_clobber(ptr noalias %A, ptr noalias %Out, i64 %N) {
; CHECK-NEXT: store i8 7, ptr [[P_NEXT]], align 1
; CHECK-NEXT: store i32 42, ptr [[P_NEXT]], align 4
; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[Z:%.*]] = add i32 [[STORE_FORWARDED]], 1
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 1
; CHECK-NEXT: [[RESULT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[I]]
; CHECK-NEXT: store i32 [[Z]], ptr [[RESULT]], align 4
; CHECK-NEXT: [[C:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
More information about the llvm-commits
mailing list