[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 03:32:13 PDT 2026
https://github.com/mbhade-amd updated https://github.com/llvm/llvm-project/pull/199606
>From 4687e85e80a2866a8c61dae7de061d9da22bf71b Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Mon, 1 Jun 2026 11:59:35 +0530
Subject: [PATCH 1/2] [SLP] Bail out on store-to-load forwarding hazards
Widening a chain of contiguous stores to a single vector store can break
store-to-load forwarding (STLF) when a later iteration loads from an
address that falls inside the widened store window but is misaligned to
it. On x86 the load then stalls until the store buffer drains
(typically 10-15 cycles per occurrence).
A canonical pattern is:
for (int i = 5; i < n; i += 4) {
int t = A[i-5];
A[i] = t + 1;
A[i+1] = t + 2;
A[i+2] = t + 3;
A[i+3] = t + 4;
}
SLP coalesces the four stores into one 16-byte vector store at A[i].
The load A[i-5] is 20 bytes behind the store base; 20 % 16 != 0 and
20 / 16 stays inside the store-buffer window, so forwarding fails.
This patch teaches SLP to detect that case and leave the chain scalar:
* Walk MemoryDepChecker::getDependences and filter to dependences that
touch a store from the current SLP chain. Per-chain filtering keeps
granularity that LAA's loop-wide MaxStoreLoadForwardSafeDistanceInBits
would lose when several independent chains live in the same loop.
* For each surviving (store, load) pair compute the byte distance with
SCEV anchored at Chain[0] (the widened store's base) rather than
LAA's per-pair representative, which can be any interior chain store
and would change the predicate result depending on which pair LAA
happened to record.
* Evaluate the misalignment-vs-store-buffer-window predicate via
VectorizerParams::isStoreLoadForwardingConflict. The predicate is
hoisted out of LoopAccessAnalysis so LAA's couldPreventStoreLoadForward
and SLP's gate share the exact same condition.
A new -slp-store-load-forward-check option (default on) acts as a
kill-switch for the heuristic.
Tests live in llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
(added in the NFC pre-commit patch); the run lines exercise both the
heuristic-on default (STLF-ON) and the kill-switch path (STLF-OFF).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 19 ++
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 9 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 12 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 139 +++++++-
.../X86/store-load-forward-conflict.ll | 310 +++++++++---------
5 files changed, 325 insertions(+), 164 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..a58c28c1d8895 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -46,6 +46,25 @@ struct VectorizerParams {
/// make more than this number of comparisons.
LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;
+ /// Returns true if a memory dependence at byte distance \p Distance between
+ /// a store and load (both with element size \p TypeByteSize bytes) would
+ /// prevent store-to-load forwarding when the store is widened to
+ /// \p VectorStoreSize bytes.
+ ///
+ /// The predicate fires when (a) the load is misaligned w.r.t. the widened
+ /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
+ /// (b) the conflicting store is still likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
+ /// Both LoopAccessAnalysis and SLPVectorizer use this as their core STLF
+ /// cost-model gate.
+ inline static bool isStoreLoadForwardingConflict(uint64_t Distance,
+ uint64_t VectorStoreSize,
+ uint64_t TypeByteSize) {
+ const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
+ return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
+ Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ }
+
// When creating runtime checks for nested loops, where possible try to
// write the checks in a form that allows them to be easily hoisted out of
// the outermost loop. For example, we can do this by expanding the range of
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..b96702a634739 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -37,6 +37,7 @@ class GetElementPtrInst;
class InsertElementInst;
class InsertValueInst;
class Instruction;
+class LoopAccessInfoManager;
class LoopInfo;
class OptimizationRemarkEmitter;
class PHINode;
@@ -70,6 +71,7 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
DominatorTree *DT = nullptr;
AssumptionCache *AC = nullptr;
DemandedBits *DB = nullptr;
+ LoopAccessInfoManager *LAIs = nullptr;
const DataLayout *DL = nullptr;
public:
@@ -80,7 +82,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
TargetTransformInfo *TTI_, TargetLibraryInfo *TLI_,
AAResults *AA_, LoopInfo *LI_, DominatorTree *DT_,
AssumptionCache *AC_, DemandedBits *DB_,
- OptimizationRemarkEmitter *ORE_);
+ OptimizationRemarkEmitter *ORE_,
+ LoopAccessInfoManager *LAIs_);
private:
/// Collect store and getelementptr instructions and organize them
@@ -171,6 +174,10 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
+ /// Returns true if vectorizing the store chain would cause store-to-load
+ /// forwarding conflicts due to short loop-carried dependence distances.
+ bool hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain, unsigned VF);
+
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 60e15b2a5bd82..0cc9490448997 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1898,20 +1898,18 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
// place. Vectorizing in such cases does not make sense.
// Store-load forwarding distance.
- // After this many iterations store-to-load forwarding conflicts should not
- // cause any slowdowns.
- const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
// Maximum vector factor.
uint64_t MaxVFWithoutSLForwardIssuesPowerOf2 =
std::min(VectorizerParams::MaxVectorWidth * TypeByteSize,
MaxStoreLoadForwardSafeDistanceInBits);
- // Compute the smallest VF at which the store and load would be misaligned.
+ // Compute the smallest VF at which the store and load would be misaligned
+ // and recent enough to still be in the store buffer. The predicate is
+ // shared with SLP via VectorizerParams::isStoreLoadForwardingConflict.
for (uint64_t VF = 2 * TypeByteSize;
VF <= MaxVFWithoutSLForwardIssuesPowerOf2; VF *= 2) {
- // If the number of vector iteration between the store and the load are
- // small we could incur conflicts.
- if (Distance % VF && Distance / VF < NumItersForStoreLoadThroughMemory) {
+ if (VectorizerParams::isStoreLoadForwardingConflict(Distance, VF,
+ TypeByteSize)) {
MaxVFWithoutSLForwardIssuesPowerOf2 = (VF >> 1);
break;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 362087c950b7b..26e8d00f7b780 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -272,6 +272,10 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
+static cl::opt<bool> EnableSLPStoreLoadForwardCheck(
+ "slp-store-load-forward-check", cl::init(true), cl::Hidden,
+ cl::desc("Check for store-to-load forwarding conflicts in SLP"));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -27292,13 +27296,18 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
auto *AC = &AM.getResult<AssumptionAnalysis>(F);
auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ auto *LAIs = &AM.getResult<LoopAccessAnalysis>(F);
- bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
+ bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE, LAIs);
if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
+ // SLP rewrites instructions but does not restructure loops, so LAA's
+ // per-loop dependence summaries and runtime checks remain valid for
+ // downstream consumers. Mirrors LoopVectorize's preservation policy.
+ PA.preserve<LoopAccessAnalysis>();
return PA;
}
@@ -27307,7 +27316,8 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
TargetLibraryInfo *TLI_, AAResults *AA_,
LoopInfo *LI_, DominatorTree *DT_,
AssumptionCache *AC_, DemandedBits *DB_,
- OptimizationRemarkEmitter *ORE_) {
+ OptimizationRemarkEmitter *ORE_,
+ LoopAccessInfoManager *LAIs_) {
if (!RunSLPVectorization)
return false;
SE = SE_;
@@ -27318,6 +27328,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
DT = DT_;
AC = AC_;
DB = DB_;
+ LAIs = LAIs_;
DL = &F.getDataLayout();
Stores.clear();
@@ -27384,6 +27395,115 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
return Changed;
}
+bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
+ unsigned VF) {
+ if (Chain.empty() || !LAIs)
+ return false;
+
+ auto *FirstStore = cast<StoreInst>(Chain[0]);
+ Loop *L = LI->getLoopFor(FirstStore->getParent());
+ if (!L) {
+ // STLF stalls only matter inside loops where the store buffer stays hot
+ // across iterations.
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: stores not in a loop, skipping\n");
+ return false;
+ }
+
+ Type *ValueTy = FirstStore->getValueOperand()->getType();
+ TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+ // SLP only handles fixed-width vectors; bail out on scalable types.
+ if (StoreSize.isScalable())
+ return false;
+ uint64_t ElementSize = StoreSize.getFixedValue();
+ if (ElementSize == 0)
+ return false;
+ uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+ << " ElementSize=" << ElementSize
+ << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+ // Build a quick lookup of the chain's stores so we can pick out only the
+ // dependences that actually involve this chain. LAA computes a single
+ // per-loop store-load forwarding cap that is the minimum across *every*
+ // dep in the loop, which over-restricts when the loop contains multiple
+ // independent SLP chains. Iterating per-dep keeps us chain-granular.
+ SmallPtrSet<const Instruction *, 8> ChainStores;
+ for (Value *V : Chain)
+ if (auto *I = dyn_cast<Instruction>(V))
+ ChainStores.insert(I);
+ if (ChainStores.empty())
+ return false;
+
+ // LoopAccessInfo is computed lazily and cached per loop, so subsequent
+ // queries for chains in the same loop are O(1) (same machinery the loop
+ // vectorizer consumes).
+ const LoopAccessInfo &LAI = LAIs->getInfo(*L);
+ const MemoryDepChecker &Dep = LAI.getDepChecker();
+ const auto *Deps = Dep.getDependences();
+ if (!Deps) {
+ // LAA bailed (volatile load, indirect access, exceeded MaxDependences,
+ // ...). Be conservative and don't reject vectorization on the SLP side.
+ LLVM_DEBUG(dbgs() << "SLP: STLF: no LAA dependence list available\n");
+ return false;
+ }
+
+ // Avoid repeating the predicate work for loads that LAA reports against
+ // multiple chain stores in turn.
+ SmallPtrSet<const LoadInst *, 8> SeenLoads;
+ for (const MemoryDepChecker::Dependence &D : *Deps) {
+ Instruction *Src = D.getSource(Dep);
+ Instruction *Dst = D.getDestination(Dep);
+
+ // Only inspect deps where one end is a store from this chain and the
+ // other end is a (simple) load.
+ bool SrcInChain = ChainStores.contains(Src);
+ bool DstInChain = ChainStores.contains(Dst);
+ if (!SrcInChain && !DstInChain)
+ continue;
+ auto *LoadI = dyn_cast<LoadInst>(SrcInChain ? Dst : Src);
+ if (!LoadI || !LoadI->isSimple() || !SeenLoads.insert(LoadI).second)
+ continue;
+
+ // Compute distance from the chain's leading edge (Chain[0]) rather than
+ // the specific store LAA paired with the load: LAA may pick any chain
+ // store as the dependence representative, but the predicate cares about
+ // the chain window's start.
+ std::optional<int64_t> Diff =
+ getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
+ LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
+ /*StrictCheck=*/false, /*CheckType=*/false);
+ if (!Diff) {
+ LLVM_DEBUG(dbgs() << "SLP: STLF: getPointersDiff returned nullopt\n");
+ continue;
+ }
+ // Only negative diffs (load below the chain) are backward loop-carried;
+ // forward references within an iteration don't cause STLF stalls.
+ if (*Diff >= 0)
+ continue;
+
+ uint64_t Distance = -static_cast<uint64_t>(*Diff) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF: chain-relevant dep distance=" << Distance
+ << " bytes\n");
+
+ // Loads inside the widened store window [base, base+VectorStoreBytes)
+ // don't overlap a *previous* iteration's vector store; they are
+ // forwarded normally.
+ if (Distance < VectorStoreBytes)
+ continue;
+
+ if (VectorizerParams::isStoreLoadForwardingConflict(
+ Distance, VectorStoreBytes, ElementSize)) {
+ LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: distance "
+ << Distance << " bytes, vector store width "
+ << VectorStoreBytes << " bytes, misalignment "
+ << (Distance % VectorStoreBytes) << "\n");
+ return true;
+ }
+ }
+
+ return false;
+}
+
std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
@@ -27405,6 +27525,21 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
return false;
}
+ // Early bail-out: reject chains that would suffer STLF stalls before doing
+ // expensive tree analysis (buildTree, reorder, getTreeCost, etc.).
+ if (EnableSLPStoreLoadForwardCheck &&
+ hasStoreLoadForwardingConflict(Chain, VF)) {
+ LLVM_DEBUG(dbgs() << "SLP: Skipping store chain due to potential "
+ "store-to-load forwarding conflict\n");
+ using namespace ore;
+ R.getORE()->emit(
+ OptimizationRemarkMissed(SV_NAME, "StoreLoadForwardConflict",
+ cast<StoreInst>(Chain[0]))
+ << "Stores not SLP vectorized: short loop-carried dependence "
+ "distance could cause store-to-load forwarding stalls");
+ return false;
+ }
+
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
<< "\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 245a537489d8f..2a6435305217b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -1,5 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-ON
+; RUN: opt < %s -passes=slp-vectorizer -slp-store-load-forward-check=false -S \
+; RUN: -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-OFF
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
;
; Test 1: Backward loop-carried load at misaligned distance.
@@ -19,28 +26,53 @@
; The new check should bail out and leave the four-wide chain scalar.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[T]], 3
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1, i32 2>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -92,9 +124,8 @@ for.end:
; The check should NOT fire; vectorization is allowed.
;
define void @stlf_no_conflict_backward_aligned(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_backward_aligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
@@ -157,7 +188,6 @@ for.end:
; Vectorization should proceed.
;
define void @stlf_no_conflict_forward(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_forward(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -222,7 +252,6 @@ for.end:
; store buffer is hot. Outside loops the check should not fire.
;
define void @stlf_no_conflict_no_loop(ptr noalias %A) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_no_loop(
; CHECK-SAME: ptr noalias [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -5
@@ -260,7 +289,6 @@ define void @stlf_no_conflict_no_loop(ptr noalias %A) {
; widened store, so the check should not fire.
;
define void @stlf_no_conflict_different_base(ptr noalias %A, ptr noalias %B,
-;
; CHECK-LABEL: define void @stlf_no_conflict_different_base(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -327,7 +355,6 @@ for.end:
; finds no offending load and vectorization proceeds.
;
define void @stlf_volatile_load_skipped(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_volatile_load_skipped(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -393,32 +420,63 @@ for.end:
; check fires on the first one encountered and bails out.
;
define void @stlf_multiple_conflicting_loads(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_multiple_conflicting_loads(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
-; CHECK-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
-; CHECK-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
-; CHECK-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
-; CHECK-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
-; CHECK-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-ON-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-ON-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-ON-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-ON-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-ON-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[SUM]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i32 [[SUM]], 2
+; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[SUM]], 3
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[SUM]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I2:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I2]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: store i32 [[T2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP2]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-OFF-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-OFF-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-OFF-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-OFF-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-OFF-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -471,7 +529,6 @@ for.end:
; conflict, and vectorization proceeds.
;
define void @stlf_pointer_diff_unknown(ptr noalias %A, ptr noalias %IdxA, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_pointer_diff_unknown(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[IDXA:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -541,7 +598,6 @@ for.end:
; catches this and allows vectorization.
;
define void @stlf_no_conflict_within_prev_store(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_within_prev_store(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -600,79 +656,7 @@ for.end:
}
;
-; Test 10: Loop body too large -> STLF check skipped.
-;
-; Same misaligned-distance shape as Test 1, but the third RUN line lowers
-; the SLP loop-size threshold to 10. The loop body is ~20 instructions
-; (well above 10 yet well below the default 128). With the default
-; threshold of 128 the check still runs and rejects vectorization, with
-; the STLF flag forced off vectorization is allowed, and with the
-; threshold lowered to 10 the loop is treated as large so the check is
-; skipped and vectorization is allowed.
-;
-define void @stlf_large_loop_bypassed(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_large_loop_bypassed(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %for.body
-
-for.body:
- %i = phi i64 [ 5, %entry ], [ %i.next, %for.body ]
-
- %back.idx = sub i64 %i, 5
- %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
- %t = load i32, ptr %back.gep, align 4
-
- %t1 = add nsw i32 %t, 1
- %t2 = add nsw i32 %t, 2
- %t3 = add nsw i32 %t, 3
- %t4 = add nsw i32 %t, 4
-
- %i1 = add nuw nsw i64 %i, 1
- %i2 = add nuw nsw i64 %i, 2
- %i3 = add nuw nsw i64 %i, 3
-
- %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
- %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
- %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
- %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
-
- store i32 %t1, ptr %gep0, align 4
- store i32 %t2, ptr %gep1, align 4
- store i32 %t3, ptr %gep2, align 4
- store i32 %t4, ptr %gep3, align 4
-
- %i.next = add nuw nsw i64 %i, 4
- %cmp = icmp slt i64 %i.next, %n
- br i1 %cmp, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
-;
-; Test 11: Distance just past NumItersForSafety boundary.
+; Test 10: Distance just past NumItersForSafety boundary.
;
; For i32 with VF=4: ElementSize=4, VectorStoreBytes=16,
; NumItersForSafety = 8 * ElementSize = 32. The check fires only when
@@ -682,7 +666,6 @@ for.end:
; (misaligned). Vectorization proceeds.
;
define void @stlf_no_conflict_iter_safety_boundary(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_iter_safety_boundary(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -741,7 +724,7 @@ for.end:
}
;
-; Test 12: Non-i32 element type (i64 VF=2).
+; Test 11: Non-i32 element type (i64 VF=2).
;
; Verifies the byte-distance math works for wider types. With i64
; (ElementSize=8) and VF=2: VectorStoreBytes=16, NumItersForSafety=64.
@@ -749,28 +732,47 @@ for.end:
; 24%16=8 != 0, so the check fires.
;
define void @stlf_conflict_backward_misaligned_i64(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned_i64(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
-; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i64 [[T]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i64 [[T]], 2
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: store i64 [[T1]], ptr [[GEP0]], align 8
+; STLF-ON-NEXT: store i64 [[T2]], ptr [[GEP1]], align 8
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
+; STLF-OFF-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
>From 52855d752960bacb9ce876a6f808fa8442601da5 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 11 Jun 2026 15:52:35 +0530
Subject: [PATCH 2/2] [SLP] Address review comments on STLF bail-out
Reworks the store-to-load forwarding gate added in the previous commit to
address review feedback on correctness and compile-time.
* Move isStoreLoadForwardingConflict from VectorizerParams into
MemoryDepChecker, so the predicate lives next to LAA's own
couldPreventStoreLoadForward and both consumers share one definition.
* Add a per-(chain-base store, VF) result cache and a cheap early-gate
that scans the loop for a simple load sharing the chain's underlying
object before LAA is ever queried, avoiding the analysis on loops that
cannot conflict and on repeated retries at different vector factors.
* Drop the per-chain ChainStores filtering. LAA records a single
representative (store, load) pair per dependence and the chosen store
is an internal detail, so candidate loads are now collected by
underlying object across the STLF-relevant dependence types
(Backward, BackwardVectorizable,
BackwardVectorizableButPreventsForwarding, ForwardButPreventsForwarding)
instead of requiring the representative store to be in this chain.
* Guard dependence endpoints against the set of instructions still live
in the loop; LAA's cached dependence list can hold pointers that
dangle after earlier SLP runs delete instructions.
* Compute the load distance with StrictCheck=true and drop the
Distance < VectorStoreBytes short-circuit, which discarded loads that
legitimately straddle the widened store window.
* Stop preserving LoopAccessAnalysis in the pass's PreservedAnalyses.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 38 ++--
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 6 +
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 5 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 163 +++++++++++-------
.../X86/store-load-forward-conflict.ll | 93 ++++++----
5 files changed, 184 insertions(+), 121 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index a58c28c1d8895..6c8ad62e65f45 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -46,25 +46,6 @@ struct VectorizerParams {
/// make more than this number of comparisons.
LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;
- /// Returns true if a memory dependence at byte distance \p Distance between
- /// a store and load (both with element size \p TypeByteSize bytes) would
- /// prevent store-to-load forwarding when the store is widened to
- /// \p VectorStoreSize bytes.
- ///
- /// The predicate fires when (a) the load is misaligned w.r.t. the widened
- /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
- /// (b) the conflicting store is still likely to be in the store buffer
- /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
- /// Both LoopAccessAnalysis and SLPVectorizer use this as their core STLF
- /// cost-model gate.
- inline static bool isStoreLoadForwardingConflict(uint64_t Distance,
- uint64_t VectorStoreSize,
- uint64_t TypeByteSize) {
- const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
- return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
- Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
- }
-
// When creating runtime checks for nested loops, where possible try to
// write the checks in a form that allows them to be easily hoisted out of
// the outermost loop. For example, we can do this by expanding the range of
@@ -253,6 +234,25 @@ class MemoryDepChecker {
std::numeric_limits<uint64_t>::max();
}
+ /// Returns true if a memory dependence at byte distance \p Distance between
+ /// a store and load (both with element size \p TypeByteSize bytes) would
+ /// prevent store-to-load forwarding when the store is widened to
+ /// \p VectorStoreSize bytes.
+ ///
+ /// The predicate fires when (a) the load is misaligned w.r.t. the widened
+ /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
+ /// (b) the conflicting store is still likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
+ /// Both couldPreventStoreLoadForward and SLPVectorizer use this as their
+ /// core STLF cost-model gate.
+ static bool isStoreLoadForwardingConflict(uint64_t Distance,
+ uint64_t VectorStoreSize,
+ uint64_t TypeByteSize) {
+ const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
+ return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
+ Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ }
+
/// Return safe power-of-2 number of elements, which do not prevent store-load
/// forwarding, multiplied by the size of the elements in bits.
uint64_t getStoreLoadForwardSafeDistanceInBits() const {
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index b96702a634739..245cf461b2198 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -19,6 +19,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
@@ -183,6 +184,11 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
+ /// Cached STLF conflict decisions keyed by (first store in chain, VF).
+ /// Avoids re-walking the LAA dependence list when the same chain is retried
+ /// at multiple vector factors.
+ DenseMap<std::pair<const StoreInst *, unsigned>, bool> StlfConflictCache;
+
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 0cc9490448997..f884d318b392b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1905,11 +1905,10 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
// Compute the smallest VF at which the store and load would be misaligned
// and recent enough to still be in the store buffer. The predicate is
- // shared with SLP via VectorizerParams::isStoreLoadForwardingConflict.
+ // shared with SLP via MemoryDepChecker::isStoreLoadForwardingConflict.
for (uint64_t VF = 2 * TypeByteSize;
VF <= MaxVFWithoutSLForwardIssuesPowerOf2; VF *= 2) {
- if (VectorizerParams::isStoreLoadForwardingConflict(Distance, VF,
- TypeByteSize)) {
+ if (isStoreLoadForwardingConflict(Distance, VF, TypeByteSize)) {
MaxVFWithoutSLForwardIssuesPowerOf2 = (VF >> 1);
break;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 26e8d00f7b780..4e128aaba8505 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27304,10 +27304,6 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
- // SLP rewrites instructions but does not restructure loops, so LAA's
- // per-loop dependence summaries and runtime checks remain valid for
- // downstream consumers. Mirrors LoopVectorize's preservation policy.
- PA.preserve<LoopAccessAnalysis>();
return PA;
}
@@ -27333,6 +27329,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
Stores.clear();
GEPs.clear();
+ StlfConflictCache.clear();
bool Changed = false;
// If the target claims to have no vector registers don't attempt
@@ -27401,107 +27398,139 @@ bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
return false;
auto *FirstStore = cast<StoreInst>(Chain[0]);
+
+ // Cache lookup: avoid re-walking the LAA dependence list when the same
+ // chain is retried at multiple vector factors.
+ auto Key = std::make_pair(FirstStore, VF);
+ auto CacheIt = StlfConflictCache.find(Key);
+ if (CacheIt != StlfConflictCache.end())
+ return CacheIt->second;
+
+ auto CacheAndReturn = [&](bool Result) -> bool {
+ StlfConflictCache[Key] = Result;
+ return Result;
+ };
+
Loop *L = LI->getLoopFor(FirstStore->getParent());
- if (!L) {
- // STLF stalls only matter inside loops where the store buffer stays hot
- // across iterations.
- LLVM_DEBUG(dbgs() << "SLP: STLF check: stores not in a loop, skipping\n");
- return false;
- }
+ if (!L)
+ return CacheAndReturn(false);
Type *ValueTy = FirstStore->getValueOperand()->getType();
TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
- // SLP only handles fixed-width vectors; bail out on scalable types.
if (StoreSize.isScalable())
- return false;
+ return CacheAndReturn(false);
uint64_t ElementSize = StoreSize.getFixedValue();
if (ElementSize == 0)
- return false;
+ return CacheAndReturn(false);
uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
<< " ElementSize=" << ElementSize
<< " VectorStoreBytes=" << VectorStoreBytes << "\n");
- // Build a quick lookup of the chain's stores so we can pick out only the
- // dependences that actually involve this chain. LAA computes a single
- // per-loop store-load forwarding cap that is the minimum across *every*
- // dep in the loop, which over-restricts when the loop contains multiple
- // independent SLP chains. Iterating per-dep keeps us chain-granular.
- SmallPtrSet<const Instruction *, 8> ChainStores;
- for (Value *V : Chain)
- if (auto *I = dyn_cast<Instruction>(V))
- ChainStores.insert(I);
- if (ChainStores.empty())
- return false;
+ // Cheap early-gate: if the loop has no loads from the same underlying
+ // object as the store chain, STLF conflicts are impossible. This avoids
+ // paying for LAA on loops that obviously cannot conflict.
+ Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+ bool HasSameBaseLoad = false;
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *LdI = dyn_cast<LoadInst>(&I)) {
+ if (LdI->isSimple() &&
+ getUnderlyingObject(LdI->getPointerOperand()) == StoreBase) {
+ HasSameBaseLoad = true;
+ break;
+ }
+ }
+ }
+ if (HasSameBaseLoad)
+ break;
+ }
+ if (!HasSameBaseLoad)
+ return CacheAndReturn(false);
- // LoopAccessInfo is computed lazily and cached per loop, so subsequent
- // queries for chains in the same loop are O(1) (same machinery the loop
- // vectorizer consumes).
const LoopAccessInfo &LAI = LAIs->getInfo(*L);
const MemoryDepChecker &Dep = LAI.getDepChecker();
const auto *Deps = Dep.getDependences();
- if (!Deps) {
- // LAA bailed (volatile load, indirect access, exceeded MaxDependences,
- // ...). Be conservative and don't reject vectorization on the SLP side.
- LLVM_DEBUG(dbgs() << "SLP: STLF: no LAA dependence list available\n");
- return false;
- }
-
- // Avoid repeating the predicate work for loads that LAA reports against
- // multiple chain stores in turn.
- SmallPtrSet<const LoadInst *, 8> SeenLoads;
+ if (!Deps)
+ return CacheAndReturn(false);
+
+ // Collect the set of instructions still live in the loop. LAA's cached
+ // dependence list holds raw pointers that may dangle after earlier SLP
+ // vectorization passes delete instructions from the same loop.
+ SmallPtrSet<const Instruction *, 32> LiveInsts;
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB)
+ LiveInsts.insert(&I);
+
+ // Pass 1: collect candidate loads from STLF-relevant dependences.
+ // LAA records one representative (store, load) pair per dependence —
+ // the chosen store is an internal LAA detail, so we must not require
+ // the representative to be a member of this particular chain. Instead,
+ // collect every simple load that appears in a backward / STLF-preventing
+ // dep and accesses the same underlying object as the chain. The
+ // chain-specific distance check happens in pass 2.
+ using DepType = MemoryDepChecker::Dependence::DepType;
+ SmallPtrSet<LoadInst *, 8> CandidateLoads;
for (const MemoryDepChecker::Dependence &D : *Deps) {
+ switch (D.Type) {
+ case DepType::Backward:
+ case DepType::BackwardVectorizable:
+ case DepType::BackwardVectorizableButPreventsForwarding:
+ case DepType::ForwardButPreventsForwarding:
+ break;
+ default:
+ continue;
+ }
+
Instruction *Src = D.getSource(Dep);
Instruction *Dst = D.getDestination(Dep);
- // Only inspect deps where one end is a store from this chain and the
- // other end is a (simple) load.
- bool SrcInChain = ChainStores.contains(Src);
- bool DstInChain = ChainStores.contains(Dst);
- if (!SrcInChain && !DstInChain)
+ if (!LiveInsts.contains(Src) || !LiveInsts.contains(Dst))
continue;
- auto *LoadI = dyn_cast<LoadInst>(SrcInChain ? Dst : Src);
- if (!LoadI || !LoadI->isSimple() || !SeenLoads.insert(LoadI).second)
+
+ // Either side may be the load; pick whichever is a LoadInst.
+ LoadInst *LoadI = dyn_cast<LoadInst>(Src);
+ if (!LoadI)
+ LoadI = dyn_cast<LoadInst>(Dst);
+ if (!LoadI || !LoadI->isSimple())
+ continue;
+
+ if (getUnderlyingObject(LoadI->getPointerOperand()) != StoreBase)
continue;
- // Compute distance from the chain's leading edge (Chain[0]) rather than
- // the specific store LAA paired with the load: LAA may pick any chain
- // store as the dependence representative, but the predicate cares about
- // the chain window's start.
+ CandidateLoads.insert(LoadI);
+ }
+
+ if (CandidateLoads.empty())
+ return CacheAndReturn(false);
+
+ // Pass 2: for each candidate load, compute the byte distance from the
+ // vector store base (Chain[0]). After SLP, the chain becomes a single
+ // wide store at Chain[0]; the STLF predicate asks whether the load
+ // straddles two of those wide stores.
+ for (LoadInst *LoadI : CandidateLoads) {
std::optional<int64_t> Diff =
getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
- /*StrictCheck=*/false, /*CheckType=*/false);
- if (!Diff) {
- LLVM_DEBUG(dbgs() << "SLP: STLF: getPointersDiff returned nullopt\n");
- continue;
- }
- // Only negative diffs (load below the chain) are backward loop-carried;
- // forward references within an iteration don't cause STLF stalls.
- if (*Diff >= 0)
+ /*StrictCheck=*/true, /*CheckType=*/false);
+ if (!Diff || *Diff >= 0)
continue;
uint64_t Distance = -static_cast<uint64_t>(*Diff) * ElementSize;
- LLVM_DEBUG(dbgs() << "SLP: STLF: chain-relevant dep distance=" << Distance
- << " bytes\n");
+ LLVM_DEBUG(dbgs() << "SLP: STLF: load=" << *LoadI << " distance="
+ << Distance << " bytes from chain base\n");
- // Loads inside the widened store window [base, base+VectorStoreBytes)
- // don't overlap a *previous* iteration's vector store; they are
- // forwarded normally.
- if (Distance < VectorStoreBytes)
- continue;
-
- if (VectorizerParams::isStoreLoadForwardingConflict(
+ if (MemoryDepChecker::isStoreLoadForwardingConflict(
Distance, VectorStoreBytes, ElementSize)) {
LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: distance "
<< Distance << " bytes, vector store width "
<< VectorStoreBytes << " bytes, misalignment "
<< (Distance % VectorStoreBytes) << "\n");
- return true;
+ return CacheAndReturn(true);
}
}
- return false;
+ return CacheAndReturn(false);
}
std::optional<bool>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 2a6435305217b..988ca11ccee6e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -23,7 +23,8 @@ target triple = "x86_64-unknown-linux-gnu"
; store at A[i]. The load at A[i-5] is 20 bytes behind:
; 20 % 16 = 4 -> misaligned, load straddles two pending vector stores
; 20 / 16 = 1 -> store still hot in store buffer
-; The new check should bail out and leave the four-wide chain scalar.
+; The check bails out at VF=4. Sub-chains at VF=2 are checked per-base:
+; {A[i+1],A[i+2]} has distance 24, 24%8=0 → safe, so it still vectorizes.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
@@ -35,18 +36,18 @@ define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[T]], 3
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
-; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
-; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1, i32 2>
-; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
@@ -589,34 +590,62 @@ for.end:
}
;
-; Test 9: Distance < VectorStoreBytes (load fully within previous vector
-; store).
+; Test 9: Motivating example — a[i] = a[i-1] + 1.
;
-; The backward load is only 1 element (4 bytes) before the stores; the entire
-; load lies inside the previous iteration's 16-byte vector store, so STLF
-; can forward. The check's `Distance < VectorStoreBytes` early-continue path
-; catches this and allows vectorization.
+; Distance = 4 bytes (1 element) from chain base. VectorStoreBytes = 16
+; at VF=4. 4 % 16 = 4 → misaligned → STLF conflict. The full 4-wide
+; chain is rejected. Sub-chains that are individually misaligned are also
+; rejected; only sub-chains where the distance happens to be aligned to
+; the narrower vector store width survive.
;
-define void @stlf_no_conflict_within_prev_store(ptr noalias %A, i64 %n) {
-; CHECK-LABEL: define void @stlf_no_conflict_within_prev_store(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+define void @stlf_conflict_short_backward(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @stlf_conflict_short_backward(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_short_backward(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
More information about the llvm-commits
mailing list