[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 23:28:43 PDT 2026
https://github.com/mbhade-amd updated https://github.com/llvm/llvm-project/pull/199606
>From 4687e85e80a2866a8c61dae7de061d9da22bf71b Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Mon, 1 Jun 2026 11:59:35 +0530
Subject: [PATCH 1/3] [SLP] Bail out on store-to-load forwarding hazards
Widening a chain of contiguous stores to a single vector store can break
store-to-load forwarding (STLF) when a later iteration loads from an
address that falls inside the widened store window but is misaligned to
it. On x86 the load then stalls until the store buffer drains
(typically 10-15 cycles per occurrence).
A canonical pattern is:
for (int i = 5; i < n; i += 4) {
int t = A[i-5];
A[i] = t + 1;
A[i+1] = t + 2;
A[i+2] = t + 3;
A[i+3] = t + 4;
}
SLP coalesces the four stores into one 16-byte vector store at A[i].
The load A[i-5] is 20 bytes behind the store base; 20 % 16 != 0 and
20 / 16 stays inside the store-buffer window, so forwarding fails.
This patch teaches SLP to detect that case and leave the chain scalar:
* Walk MemoryDepChecker::getDependences and filter to dependences that
touch a store from the current SLP chain. Per-chain filtering keeps
granularity that LAA's loop-wide MaxStoreLoadForwardSafeDistanceInBits
would lose when several independent chains live in the same loop.
* For each surviving (store, load) pair compute the byte distance with
SCEV anchored at Chain[0] (the widened store's base) rather than
LAA's per-pair representative, which can be any interior chain store
and would change the predicate result depending on which pair LAA
happened to record.
* Evaluate the misalignment-vs-store-buffer-window predicate via
VectorizerParams::isStoreLoadForwardingConflict. The predicate is
hoisted out of LoopAccessAnalysis so LAA's couldPreventStoreLoadForward
and SLP's gate share the exact same condition.
A new -slp-store-load-forward-check option (default on) acts as a
kill-switch for the heuristic.
Tests live in llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
(added in the NFC pre-commit patch); the run lines exercise both the
heuristic-on default (STLF-ON) and the kill-switch path (STLF-OFF).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 19 ++
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 9 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 12 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 139 +++++++-
.../X86/store-load-forward-conflict.ll | 310 +++++++++---------
5 files changed, 325 insertions(+), 164 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..a58c28c1d8895 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -46,6 +46,25 @@ struct VectorizerParams {
/// make more than this number of comparisons.
LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;
+ /// Returns true if a memory dependence at byte distance \p Distance between
+ /// a store and load (both with element size \p TypeByteSize bytes) would
+ /// prevent store-to-load forwarding when the store is widened to
+ /// \p VectorStoreSize bytes.
+ ///
+ /// The predicate fires when (a) the load is misaligned w.r.t. the widened
+ /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
+ /// (b) the conflicting store is still likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
+ /// Both LoopAccessAnalysis and SLPVectorizer use this as their core STLF
+ /// cost-model gate.
+ inline static bool isStoreLoadForwardingConflict(uint64_t Distance,
+ uint64_t VectorStoreSize,
+ uint64_t TypeByteSize) {
+ const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
+ return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
+ Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ }
+
// When creating runtime checks for nested loops, where possible try to
// write the checks in a form that allows them to be easily hoisted out of
// the outermost loop. For example, we can do this by expanding the range of
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..b96702a634739 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -37,6 +37,7 @@ class GetElementPtrInst;
class InsertElementInst;
class InsertValueInst;
class Instruction;
+class LoopAccessInfoManager;
class LoopInfo;
class OptimizationRemarkEmitter;
class PHINode;
@@ -70,6 +71,7 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
DominatorTree *DT = nullptr;
AssumptionCache *AC = nullptr;
DemandedBits *DB = nullptr;
+ LoopAccessInfoManager *LAIs = nullptr;
const DataLayout *DL = nullptr;
public:
@@ -80,7 +82,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
TargetTransformInfo *TTI_, TargetLibraryInfo *TLI_,
AAResults *AA_, LoopInfo *LI_, DominatorTree *DT_,
AssumptionCache *AC_, DemandedBits *DB_,
- OptimizationRemarkEmitter *ORE_);
+ OptimizationRemarkEmitter *ORE_,
+ LoopAccessInfoManager *LAIs_);
private:
/// Collect store and getelementptr instructions and organize them
@@ -171,6 +174,10 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
+ /// Returns true if vectorizing the store chain would cause store-to-load
+ /// forwarding conflicts due to short loop-carried dependence distances.
+ bool hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain, unsigned VF);
+
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 60e15b2a5bd82..0cc9490448997 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1898,20 +1898,18 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
// place. Vectorizing in such cases does not make sense.
// Store-load forwarding distance.
- // After this many iterations store-to-load forwarding conflicts should not
- // cause any slowdowns.
- const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
// Maximum vector factor.
uint64_t MaxVFWithoutSLForwardIssuesPowerOf2 =
std::min(VectorizerParams::MaxVectorWidth * TypeByteSize,
MaxStoreLoadForwardSafeDistanceInBits);
- // Compute the smallest VF at which the store and load would be misaligned.
+ // Compute the smallest VF at which the store and load would be misaligned
+ // and recent enough to still be in the store buffer. The predicate is
+ // shared with SLP via VectorizerParams::isStoreLoadForwardingConflict.
for (uint64_t VF = 2 * TypeByteSize;
VF <= MaxVFWithoutSLForwardIssuesPowerOf2; VF *= 2) {
- // If the number of vector iteration between the store and the load are
- // small we could incur conflicts.
- if (Distance % VF && Distance / VF < NumItersForStoreLoadThroughMemory) {
+ if (VectorizerParams::isStoreLoadForwardingConflict(Distance, VF,
+ TypeByteSize)) {
MaxVFWithoutSLForwardIssuesPowerOf2 = (VF >> 1);
break;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 362087c950b7b..26e8d00f7b780 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -272,6 +272,10 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
+static cl::opt<bool> EnableSLPStoreLoadForwardCheck(
+ "slp-store-load-forward-check", cl::init(true), cl::Hidden,
+ cl::desc("Check for store-to-load forwarding conflicts in SLP"));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -27292,13 +27296,18 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
auto *AC = &AM.getResult<AssumptionAnalysis>(F);
auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ auto *LAIs = &AM.getResult<LoopAccessAnalysis>(F);
- bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
+ bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE, LAIs);
if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
+ // SLP rewrites instructions but does not restructure loops, so LAA's
+ // per-loop dependence summaries and runtime checks remain valid for
+ // downstream consumers. Mirrors LoopVectorize's preservation policy.
+ PA.preserve<LoopAccessAnalysis>();
return PA;
}
@@ -27307,7 +27316,8 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
TargetLibraryInfo *TLI_, AAResults *AA_,
LoopInfo *LI_, DominatorTree *DT_,
AssumptionCache *AC_, DemandedBits *DB_,
- OptimizationRemarkEmitter *ORE_) {
+ OptimizationRemarkEmitter *ORE_,
+ LoopAccessInfoManager *LAIs_) {
if (!RunSLPVectorization)
return false;
SE = SE_;
@@ -27318,6 +27328,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
DT = DT_;
AC = AC_;
DB = DB_;
+ LAIs = LAIs_;
DL = &F.getDataLayout();
Stores.clear();
@@ -27384,6 +27395,115 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
return Changed;
}
+bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
+ unsigned VF) {
+ if (Chain.empty() || !LAIs)
+ return false;
+
+ auto *FirstStore = cast<StoreInst>(Chain[0]);
+ Loop *L = LI->getLoopFor(FirstStore->getParent());
+ if (!L) {
+ // STLF stalls only matter inside loops where the store buffer stays hot
+ // across iterations.
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: stores not in a loop, skipping\n");
+ return false;
+ }
+
+ Type *ValueTy = FirstStore->getValueOperand()->getType();
+ TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+ // SLP only handles fixed-width vectors; bail out on scalable types.
+ if (StoreSize.isScalable())
+ return false;
+ uint64_t ElementSize = StoreSize.getFixedValue();
+ if (ElementSize == 0)
+ return false;
+ uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+ << " ElementSize=" << ElementSize
+ << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+ // Build a quick lookup of the chain's stores so we can pick out only the
+ // dependences that actually involve this chain. LAA computes a single
+ // per-loop store-load forwarding cap that is the minimum across *every*
+ // dep in the loop, which over-restricts when the loop contains multiple
+ // independent SLP chains. Iterating per-dep keeps us chain-granular.
+ SmallPtrSet<const Instruction *, 8> ChainStores;
+ for (Value *V : Chain)
+ if (auto *I = dyn_cast<Instruction>(V))
+ ChainStores.insert(I);
+ if (ChainStores.empty())
+ return false;
+
+ // LoopAccessInfo is computed lazily and cached per loop, so subsequent
+ // queries for chains in the same loop are O(1) (same machinery the loop
+ // vectorizer consumes).
+ const LoopAccessInfo &LAI = LAIs->getInfo(*L);
+ const MemoryDepChecker &Dep = LAI.getDepChecker();
+ const auto *Deps = Dep.getDependences();
+ if (!Deps) {
+ // LAA bailed (volatile load, indirect access, exceeded MaxDependences,
+ // ...). Be conservative and don't reject vectorization on the SLP side.
+ LLVM_DEBUG(dbgs() << "SLP: STLF: no LAA dependence list available\n");
+ return false;
+ }
+
+ // Avoid repeating the predicate work for loads that LAA reports against
+ // multiple chain stores in turn.
+ SmallPtrSet<const LoadInst *, 8> SeenLoads;
+ for (const MemoryDepChecker::Dependence &D : *Deps) {
+ Instruction *Src = D.getSource(Dep);
+ Instruction *Dst = D.getDestination(Dep);
+
+ // Only inspect deps where one end is a store from this chain and the
+ // other end is a (simple) load.
+ bool SrcInChain = ChainStores.contains(Src);
+ bool DstInChain = ChainStores.contains(Dst);
+ if (!SrcInChain && !DstInChain)
+ continue;
+ auto *LoadI = dyn_cast<LoadInst>(SrcInChain ? Dst : Src);
+ if (!LoadI || !LoadI->isSimple() || !SeenLoads.insert(LoadI).second)
+ continue;
+
+ // Compute distance from the chain's leading edge (Chain[0]) rather than
+ // the specific store LAA paired with the load: LAA may pick any chain
+ // store as the dependence representative, but the predicate cares about
+ // the chain window's start.
+ std::optional<int64_t> Diff =
+ getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
+ LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
+ /*StrictCheck=*/false, /*CheckType=*/false);
+ if (!Diff) {
+ LLVM_DEBUG(dbgs() << "SLP: STLF: getPointersDiff returned nullopt\n");
+ continue;
+ }
+ // Only negative diffs (load below the chain) are backward loop-carried;
+ // forward references within an iteration don't cause STLF stalls.
+ if (*Diff >= 0)
+ continue;
+
+ uint64_t Distance = -static_cast<uint64_t>(*Diff) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF: chain-relevant dep distance=" << Distance
+ << " bytes\n");
+
+ // Loads inside the widened store window [base, base+VectorStoreBytes)
+ // don't overlap a *previous* iteration's vector store; they are
+ // forwarded normally.
+ if (Distance < VectorStoreBytes)
+ continue;
+
+ if (VectorizerParams::isStoreLoadForwardingConflict(
+ Distance, VectorStoreBytes, ElementSize)) {
+ LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: distance "
+ << Distance << " bytes, vector store width "
+ << VectorStoreBytes << " bytes, misalignment "
+ << (Distance % VectorStoreBytes) << "\n");
+ return true;
+ }
+ }
+
+ return false;
+}
+
std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
@@ -27405,6 +27525,21 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
return false;
}
+ // Early bail-out: reject chains that would suffer STLF stalls before doing
+ // expensive tree analysis (buildTree, reorder, getTreeCost, etc.).
+ if (EnableSLPStoreLoadForwardCheck &&
+ hasStoreLoadForwardingConflict(Chain, VF)) {
+ LLVM_DEBUG(dbgs() << "SLP: Skipping store chain due to potential "
+ "store-to-load forwarding conflict\n");
+ using namespace ore;
+ R.getORE()->emit(
+ OptimizationRemarkMissed(SV_NAME, "StoreLoadForwardConflict",
+ cast<StoreInst>(Chain[0]))
+ << "Stores not SLP vectorized: short loop-carried dependence "
+ "distance could cause store-to-load forwarding stalls");
+ return false;
+ }
+
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
<< "\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 245a537489d8f..2a6435305217b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -1,5 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-ON
+; RUN: opt < %s -passes=slp-vectorizer -slp-store-load-forward-check=false -S \
+; RUN: -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-OFF
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
;
; Test 1: Backward loop-carried load at misaligned distance.
@@ -19,28 +26,53 @@
; The new check should bail out and leave the four-wide chain scalar.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[T]], 3
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1, i32 2>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -92,9 +124,8 @@ for.end:
; The check should NOT fire; vectorization is allowed.
;
define void @stlf_no_conflict_backward_aligned(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_backward_aligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
@@ -157,7 +188,6 @@ for.end:
; Vectorization should proceed.
;
define void @stlf_no_conflict_forward(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_forward(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -222,7 +252,6 @@ for.end:
; store buffer is hot. Outside loops the check should not fire.
;
define void @stlf_no_conflict_no_loop(ptr noalias %A) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_no_loop(
; CHECK-SAME: ptr noalias [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -5
@@ -260,7 +289,6 @@ define void @stlf_no_conflict_no_loop(ptr noalias %A) {
; widened store, so the check should not fire.
;
define void @stlf_no_conflict_different_base(ptr noalias %A, ptr noalias %B,
-;
; CHECK-LABEL: define void @stlf_no_conflict_different_base(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -327,7 +355,6 @@ for.end:
; finds no offending load and vectorization proceeds.
;
define void @stlf_volatile_load_skipped(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_volatile_load_skipped(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -393,32 +420,63 @@ for.end:
; check fires on the first one encountered and bails out.
;
define void @stlf_multiple_conflicting_loads(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_multiple_conflicting_loads(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
-; CHECK-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
-; CHECK-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
-; CHECK-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
-; CHECK-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
-; CHECK-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-ON-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-ON-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-ON-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-ON-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-ON-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[SUM]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i32 [[SUM]], 2
+; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[SUM]], 3
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[SUM]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I2:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I2]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: store i32 [[T2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP2]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-OFF-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-OFF-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-OFF-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-OFF-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-OFF-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -471,7 +529,6 @@ for.end:
; conflict, and vectorization proceeds.
;
define void @stlf_pointer_diff_unknown(ptr noalias %A, ptr noalias %IdxA, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_pointer_diff_unknown(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[IDXA:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -541,7 +598,6 @@ for.end:
; catches this and allows vectorization.
;
define void @stlf_no_conflict_within_prev_store(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_within_prev_store(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -600,79 +656,7 @@ for.end:
}
;
-; Test 10: Loop body too large -> STLF check skipped.
-;
-; Same misaligned-distance shape as Test 1, but the third RUN line lowers
-; the SLP loop-size threshold to 10. The loop body is ~20 instructions
-; (well above 10 yet well below the default 128). With the default
-; threshold of 128 the check still runs and rejects vectorization, with
-; the STLF flag forced off vectorization is allowed, and with the
-; threshold lowered to 10 the loop is treated as large so the check is
-; skipped and vectorization is allowed.
-;
-define void @stlf_large_loop_bypassed(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_large_loop_bypassed(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %for.body
-
-for.body:
- %i = phi i64 [ 5, %entry ], [ %i.next, %for.body ]
-
- %back.idx = sub i64 %i, 5
- %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
- %t = load i32, ptr %back.gep, align 4
-
- %t1 = add nsw i32 %t, 1
- %t2 = add nsw i32 %t, 2
- %t3 = add nsw i32 %t, 3
- %t4 = add nsw i32 %t, 4
-
- %i1 = add nuw nsw i64 %i, 1
- %i2 = add nuw nsw i64 %i, 2
- %i3 = add nuw nsw i64 %i, 3
-
- %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
- %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
- %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
- %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
-
- store i32 %t1, ptr %gep0, align 4
- store i32 %t2, ptr %gep1, align 4
- store i32 %t3, ptr %gep2, align 4
- store i32 %t4, ptr %gep3, align 4
-
- %i.next = add nuw nsw i64 %i, 4
- %cmp = icmp slt i64 %i.next, %n
- br i1 %cmp, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
-;
-; Test 11: Distance just past NumItersForSafety boundary.
+; Test 10: Distance just past NumItersForSafety boundary.
;
; For i32 with VF=4: ElementSize=4, VectorStoreBytes=16,
; NumItersForSafety = 8 * ElementSize = 32. The check fires only when
@@ -682,7 +666,6 @@ for.end:
; (misaligned). Vectorization proceeds.
;
define void @stlf_no_conflict_iter_safety_boundary(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_iter_safety_boundary(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -741,7 +724,7 @@ for.end:
}
;
-; Test 12: Non-i32 element type (i64 VF=2).
+; Test 11: Non-i32 element type (i64 VF=2).
;
; Verifies the byte-distance math works for wider types. With i64
; (ElementSize=8) and VF=2: VectorStoreBytes=16, NumItersForSafety=64.
@@ -749,28 +732,47 @@ for.end:
; 24%16=8 != 0, so the check fires.
;
define void @stlf_conflict_backward_misaligned_i64(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned_i64(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
-; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i64 [[T]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i64 [[T]], 2
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: store i64 [[T1]], ptr [[GEP0]], align 8
+; STLF-ON-NEXT: store i64 [[T2]], ptr [[GEP1]], align 8
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
+; STLF-OFF-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
>From 52855d752960bacb9ce876a6f808fa8442601da5 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 11 Jun 2026 15:52:35 +0530
Subject: [PATCH 2/3] [SLP] Address review comments on STLF bail-out
Reworks the store-to-load forwarding gate added in the previous commit to
address review feedback on correctness and compile-time.
* Move isStoreLoadForwardingConflict from VectorizerParams into
MemoryDepChecker, so the predicate lives next to LAA's own
couldPreventStoreLoadForward and both consumers share one definition.
* Add a per-(chain-base store, VF) result cache and a cheap early-gate
that scans the loop for a simple load sharing the chain's underlying
object before LAA is ever queried, avoiding the analysis on loops that
cannot conflict and on repeated retries at different vector factors.
* Drop the per-chain ChainStores filtering. LAA records a single
representative (store, load) pair per dependence and the chosen store
is an internal detail, so candidate loads are now collected by
underlying object across the STLF-relevant dependence types
(Backward, BackwardVectorizable,
BackwardVectorizableButPreventsForwarding, ForwardButPreventsForwarding)
instead of requiring the representative store to be in this chain.
* Guard dependence endpoints against the set of instructions still live
in the loop; LAA's cached dependence list can hold pointers that
dangle after earlier SLP runs delete instructions.
* Compute the load distance with StrictCheck=true and drop the
Distance < VectorStoreBytes short-circuit, which discarded loads that
legitimately straddle the widened store window.
* Stop preserving LoopAccessAnalysis in the pass's PreservedAnalyses.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 38 ++--
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 6 +
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 5 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 163 +++++++++++-------
.../X86/store-load-forward-conflict.ll | 93 ++++++----
5 files changed, 184 insertions(+), 121 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index a58c28c1d8895..6c8ad62e65f45 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -46,25 +46,6 @@ struct VectorizerParams {
/// make more than this number of comparisons.
LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;
- /// Returns true if a memory dependence at byte distance \p Distance between
- /// a store and load (both with element size \p TypeByteSize bytes) would
- /// prevent store-to-load forwarding when the store is widened to
- /// \p VectorStoreSize bytes.
- ///
- /// The predicate fires when (a) the load is misaligned w.r.t. the widened
- /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
- /// (b) the conflicting store is still likely to be in the store buffer
- /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
- /// Both LoopAccessAnalysis and SLPVectorizer use this as their core STLF
- /// cost-model gate.
- inline static bool isStoreLoadForwardingConflict(uint64_t Distance,
- uint64_t VectorStoreSize,
- uint64_t TypeByteSize) {
- const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
- return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
- Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
- }
-
// When creating runtime checks for nested loops, where possible try to
// write the checks in a form that allows them to be easily hoisted out of
// the outermost loop. For example, we can do this by expanding the range of
@@ -253,6 +234,25 @@ class MemoryDepChecker {
std::numeric_limits<uint64_t>::max();
}
+ /// Returns true if a memory dependence at byte distance \p Distance between
+ /// a store and load (both with element size \p TypeByteSize bytes) would
+ /// prevent store-to-load forwarding when the store is widened to
+ /// \p VectorStoreSize bytes.
+ ///
+ /// The predicate fires when (a) the load is misaligned w.r.t. the widened
+ /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
+ /// (b) the conflicting store is still likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
+ /// Both couldPreventStoreLoadForward and SLPVectorizer use this as their
+ /// core STLF cost-model gate.
+ static bool isStoreLoadForwardingConflict(uint64_t Distance,
+ uint64_t VectorStoreSize,
+ uint64_t TypeByteSize) {
+ const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
+ return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
+ Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ }
+
/// Return safe power-of-2 number of elements, which do not prevent store-load
/// forwarding, multiplied by the size of the elements in bits.
uint64_t getStoreLoadForwardSafeDistanceInBits() const {
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index b96702a634739..245cf461b2198 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -19,6 +19,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
@@ -183,6 +184,11 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
+ /// Cached STLF conflict decisions keyed by (first store in chain, VF).
+ /// Avoids re-walking the LAA dependence list when the same chain is retried
+ /// at multiple vector factors.
+ DenseMap<std::pair<const StoreInst *, unsigned>, bool> StlfConflictCache;
+
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 0cc9490448997..f884d318b392b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1905,11 +1905,10 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
// Compute the smallest VF at which the store and load would be misaligned
// and recent enough to still be in the store buffer. The predicate is
- // shared with SLP via VectorizerParams::isStoreLoadForwardingConflict.
+ // shared with SLP via MemoryDepChecker::isStoreLoadForwardingConflict.
for (uint64_t VF = 2 * TypeByteSize;
VF <= MaxVFWithoutSLForwardIssuesPowerOf2; VF *= 2) {
- if (VectorizerParams::isStoreLoadForwardingConflict(Distance, VF,
- TypeByteSize)) {
+ if (isStoreLoadForwardingConflict(Distance, VF, TypeByteSize)) {
MaxVFWithoutSLForwardIssuesPowerOf2 = (VF >> 1);
break;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 26e8d00f7b780..4e128aaba8505 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27304,10 +27304,6 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
- // SLP rewrites instructions but does not restructure loops, so LAA's
- // per-loop dependence summaries and runtime checks remain valid for
- // downstream consumers. Mirrors LoopVectorize's preservation policy.
- PA.preserve<LoopAccessAnalysis>();
return PA;
}
@@ -27333,6 +27329,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
Stores.clear();
GEPs.clear();
+ StlfConflictCache.clear();
bool Changed = false;
// If the target claims to have no vector registers don't attempt
@@ -27401,107 +27398,139 @@ bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
return false;
auto *FirstStore = cast<StoreInst>(Chain[0]);
+
+ // Cache lookup: avoid re-walking the LAA dependence list when the same
+ // chain is retried at multiple vector factors.
+ auto Key = std::make_pair(FirstStore, VF);
+ auto CacheIt = StlfConflictCache.find(Key);
+ if (CacheIt != StlfConflictCache.end())
+ return CacheIt->second;
+
+ auto CacheAndReturn = [&](bool Result) -> bool {
+ StlfConflictCache[Key] = Result;
+ return Result;
+ };
+
Loop *L = LI->getLoopFor(FirstStore->getParent());
- if (!L) {
- // STLF stalls only matter inside loops where the store buffer stays hot
- // across iterations.
- LLVM_DEBUG(dbgs() << "SLP: STLF check: stores not in a loop, skipping\n");
- return false;
- }
+ if (!L)
+ return CacheAndReturn(false);
Type *ValueTy = FirstStore->getValueOperand()->getType();
TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
- // SLP only handles fixed-width vectors; bail out on scalable types.
if (StoreSize.isScalable())
- return false;
+ return CacheAndReturn(false);
uint64_t ElementSize = StoreSize.getFixedValue();
if (ElementSize == 0)
- return false;
+ return CacheAndReturn(false);
uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
<< " ElementSize=" << ElementSize
<< " VectorStoreBytes=" << VectorStoreBytes << "\n");
- // Build a quick lookup of the chain's stores so we can pick out only the
- // dependences that actually involve this chain. LAA computes a single
- // per-loop store-load forwarding cap that is the minimum across *every*
- // dep in the loop, which over-restricts when the loop contains multiple
- // independent SLP chains. Iterating per-dep keeps us chain-granular.
- SmallPtrSet<const Instruction *, 8> ChainStores;
- for (Value *V : Chain)
- if (auto *I = dyn_cast<Instruction>(V))
- ChainStores.insert(I);
- if (ChainStores.empty())
- return false;
+ // Cheap early-gate: if the loop has no loads from the same underlying
+ // object as the store chain, STLF conflicts are impossible. This avoids
+ // paying for LAA on loops that obviously cannot conflict.
+ Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+ bool HasSameBaseLoad = false;
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *LdI = dyn_cast<LoadInst>(&I)) {
+ if (LdI->isSimple() &&
+ getUnderlyingObject(LdI->getPointerOperand()) == StoreBase) {
+ HasSameBaseLoad = true;
+ break;
+ }
+ }
+ }
+ if (HasSameBaseLoad)
+ break;
+ }
+ if (!HasSameBaseLoad)
+ return CacheAndReturn(false);
- // LoopAccessInfo is computed lazily and cached per loop, so subsequent
- // queries for chains in the same loop are O(1) (same machinery the loop
- // vectorizer consumes).
const LoopAccessInfo &LAI = LAIs->getInfo(*L);
const MemoryDepChecker &Dep = LAI.getDepChecker();
const auto *Deps = Dep.getDependences();
- if (!Deps) {
- // LAA bailed (volatile load, indirect access, exceeded MaxDependences,
- // ...). Be conservative and don't reject vectorization on the SLP side.
- LLVM_DEBUG(dbgs() << "SLP: STLF: no LAA dependence list available\n");
- return false;
- }
-
- // Avoid repeating the predicate work for loads that LAA reports against
- // multiple chain stores in turn.
- SmallPtrSet<const LoadInst *, 8> SeenLoads;
+ if (!Deps)
+ return CacheAndReturn(false);
+
+ // Collect the set of instructions still live in the loop. LAA's cached
+ // dependence list holds raw pointers that may dangle after earlier SLP
+ // vectorization passes delete instructions from the same loop.
+ SmallPtrSet<const Instruction *, 32> LiveInsts;
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB)
+ LiveInsts.insert(&I);
+
+ // Pass 1: collect candidate loads from STLF-relevant dependences.
+ // LAA records one representative (store, load) pair per dependence —
+ // the chosen store is an internal LAA detail, so we must not require
+ // the representative to be a member of this particular chain. Instead,
+ // collect every simple load that appears in a backward / STLF-preventing
+ // dep and accesses the same underlying object as the chain. The
+ // chain-specific distance check happens in pass 2.
+ using DepType = MemoryDepChecker::Dependence::DepType;
+ SmallPtrSet<LoadInst *, 8> CandidateLoads;
for (const MemoryDepChecker::Dependence &D : *Deps) {
+ switch (D.Type) {
+ case DepType::Backward:
+ case DepType::BackwardVectorizable:
+ case DepType::BackwardVectorizableButPreventsForwarding:
+ case DepType::ForwardButPreventsForwarding:
+ break;
+ default:
+ continue;
+ }
+
Instruction *Src = D.getSource(Dep);
Instruction *Dst = D.getDestination(Dep);
- // Only inspect deps where one end is a store from this chain and the
- // other end is a (simple) load.
- bool SrcInChain = ChainStores.contains(Src);
- bool DstInChain = ChainStores.contains(Dst);
- if (!SrcInChain && !DstInChain)
+ if (!LiveInsts.contains(Src) || !LiveInsts.contains(Dst))
continue;
- auto *LoadI = dyn_cast<LoadInst>(SrcInChain ? Dst : Src);
- if (!LoadI || !LoadI->isSimple() || !SeenLoads.insert(LoadI).second)
+
+ // Either side may be the load; pick whichever is a LoadInst.
+ LoadInst *LoadI = dyn_cast<LoadInst>(Src);
+ if (!LoadI)
+ LoadI = dyn_cast<LoadInst>(Dst);
+ if (!LoadI || !LoadI->isSimple())
+ continue;
+
+ if (getUnderlyingObject(LoadI->getPointerOperand()) != StoreBase)
continue;
- // Compute distance from the chain's leading edge (Chain[0]) rather than
- // the specific store LAA paired with the load: LAA may pick any chain
- // store as the dependence representative, but the predicate cares about
- // the chain window's start.
+ CandidateLoads.insert(LoadI);
+ }
+
+ if (CandidateLoads.empty())
+ return CacheAndReturn(false);
+
+ // Pass 2: for each candidate load, compute the byte distance from the
+ // vector store base (Chain[0]). After SLP, the chain becomes a single
+ // wide store at Chain[0]; the STLF predicate asks whether the load
+ // straddles two of those wide stores.
+ for (LoadInst *LoadI : CandidateLoads) {
std::optional<int64_t> Diff =
getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
- /*StrictCheck=*/false, /*CheckType=*/false);
- if (!Diff) {
- LLVM_DEBUG(dbgs() << "SLP: STLF: getPointersDiff returned nullopt\n");
- continue;
- }
- // Only negative diffs (load below the chain) are backward loop-carried;
- // forward references within an iteration don't cause STLF stalls.
- if (*Diff >= 0)
+ /*StrictCheck=*/true, /*CheckType=*/false);
+ if (!Diff || *Diff >= 0)
continue;
uint64_t Distance = -static_cast<uint64_t>(*Diff) * ElementSize;
- LLVM_DEBUG(dbgs() << "SLP: STLF: chain-relevant dep distance=" << Distance
- << " bytes\n");
+ LLVM_DEBUG(dbgs() << "SLP: STLF: load=" << *LoadI << " distance="
+ << Distance << " bytes from chain base\n");
- // Loads inside the widened store window [base, base+VectorStoreBytes)
- // don't overlap a *previous* iteration's vector store; they are
- // forwarded normally.
- if (Distance < VectorStoreBytes)
- continue;
-
- if (VectorizerParams::isStoreLoadForwardingConflict(
+ if (MemoryDepChecker::isStoreLoadForwardingConflict(
Distance, VectorStoreBytes, ElementSize)) {
LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: distance "
<< Distance << " bytes, vector store width "
<< VectorStoreBytes << " bytes, misalignment "
<< (Distance % VectorStoreBytes) << "\n");
- return true;
+ return CacheAndReturn(true);
}
}
- return false;
+ return CacheAndReturn(false);
}
std::optional<bool>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 2a6435305217b..988ca11ccee6e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -23,7 +23,8 @@ target triple = "x86_64-unknown-linux-gnu"
; store at A[i]. The load at A[i-5] is 20 bytes behind:
; 20 % 16 = 4 -> misaligned, load straddles two pending vector stores
; 20 / 16 = 1 -> store still hot in store buffer
-; The new check should bail out and leave the four-wide chain scalar.
+; The check bails out at VF=4. Sub-chains at VF=2 are checked per-base:
+; {A[i+1],A[i+2]} has distance 24, 24%8=0 → safe, so it still vectorizes.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
@@ -35,18 +36,18 @@ define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[T]], 3
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
-; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
-; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1, i32 2>
-; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
@@ -589,34 +590,62 @@ for.end:
}
;
-; Test 9: Distance < VectorStoreBytes (load fully within previous vector
-; store).
+; Test 9: Motivating example — a[i] = a[i-1] + 1.
;
-; The backward load is only 1 element (4 bytes) before the stores; the entire
-; load lies inside the previous iteration's 16-byte vector store, so STLF
-; can forward. The check's `Distance < VectorStoreBytes` early-continue path
-; catches this and allows vectorization.
+; Distance = 4 bytes (1 element) from chain base. VectorStoreBytes = 16
+; at VF=4. 4 % 16 = 4 → misaligned → STLF conflict. The full 4-wide
+; chain is rejected. Sub-chains that are individually misaligned are also
+; rejected; only sub-chains where the distance happens to be aligned to
+; the narrower vector store width survive.
;
-define void @stlf_no_conflict_within_prev_store(ptr noalias %A, i64 %n) {
-; CHECK-LABEL: define void @stlf_no_conflict_within_prev_store(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+define void @stlf_conflict_short_backward(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @stlf_conflict_short_backward(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_short_backward(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
>From a20a32ca56ff1f6e03c4cfaed51cdc2f94c54c5d Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Mon, 20 Jul 2026 11:55:30 +0530
Subject: [PATCH 3/3] [SLP] Model store-to-load forwarding hazards as a cost
penalty
The initial store-to-load forwarding (STLF) support made SLP bail out of
vectorizing a store chain whenever widening it could break forwarding.
A hard veto is too aggressive: it also rejects chains where the widened
store is still clearly profitable despite an occasional forwarding stall.
Replace the veto with a cost penalty so the STLF hazard participates in
the normal cost/benefit comparison instead of unconditionally blocking
vectorization:
* Add TargetTransformInfo::getStoreLoadForwardingConflictCost(), with a
BasicTTIImpl default that reads the penalty from the subtarget
scheduling model. findStoreLoadForwardingConflict() now only detects
the hazard; the SLP cost model adds the returned penalty to the
vector store cost (VecStCost) rather than returning early.
* Plumb the penalty through the scheduling model: a new
StoreLoadForwardingConflictCost field in MCSchedModel
(TargetSchedule.td, MCSchedule.{h,cpp}, SubtargetEmitter) lets a
target describe the stall cost; X86ScheduleZnver4 sets it.
* -slp-store-load-forward-check (default on) remains the kill-switch.
Tests: store-load-forward-cost.ll and store-load-forward-profitable.ll
cover the penalty value and the profitable-despite-penalty case;
store-load-forward-conflict.ll is updated for the cost-based decisions.
---
.../llvm/Analysis/TargetTransformInfo.h | 9 ++
.../llvm/Analysis/TargetTransformInfoImpl.h | 5 +
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 7 +
llvm/include/llvm/MC/MCSchedule.h | 9 ++
llvm/include/llvm/Target/TargetSchedule.td | 3 +
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 9 --
llvm/lib/Analysis/TargetTransformInfo.cpp | 15 ++
llvm/lib/MC/MCSchedule.cpp | 1 +
llvm/lib/Target/X86/X86ScheduleZnver4.td | 4 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 59 ++++---
.../X86/store-load-forward-conflict.ll | 16 +-
.../X86/store-load-forward-cost.ll | 105 +++++++++++++
.../X86/store-load-forward-profitable.ll | 145 ++++++++++++++++++
llvm/utils/TableGen/SubtargetEmitter.cpp | 1 +
14 files changed, 348 insertions(+), 40 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7d58473b81265..c12a8c1906fa4 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1404,6 +1404,15 @@ class TargetTransformInfo {
/// \return The size of a cache line in bytes.
LLVM_ABI unsigned getCacheLineSize() const;
+ /// \return The penalty for a store-to-load forwarding (STLF) hazard when a
+ /// chain of stores is widened into the vector store \p VecTy inside a loop
+ /// and a nearby loop-carried load straddles that widened store, so hardware
+ /// store-to-load forwarding fails and the load stalls until the store drains
+ /// to L1. Targets return the modeled per-occurrence penalty, or 0 if they do
+ /// not model this hazard.
+ LLVM_ABI InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetCostKind CostKind) const;
+
/// The possible cache levels
enum class CacheLevel {
L1D, // The L1 data cache
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 89fd4a1e7628e..01e7db248c072 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -676,6 +676,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getCacheLineSize() const { return 0; }
+
+ virtual InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
+ return 0;
+ }
virtual std::optional<unsigned>
getCacheSize(TargetTransformInfo::CacheLevel Level) const {
switch (Level) {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e4b6bf51c7a4e..63d4f08b1a585 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -853,6 +853,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return getST()->getCacheLineSize();
}
+ InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TTI::TargetCostKind CostKind) const override {
+ // The per-occurrence STLF stall penalty is a microarchitectural cycle
+ // count sourced from the subtarget's scheduling model (0 = not modeled).
+ return InstructionCost(getST()->getSchedModel().StoreLoadForwardingPenalty);
+ }
+
unsigned getPrefetchDistance() const override {
return getST()->getPrefetchDistance();
}
diff --git a/llvm/include/llvm/MC/MCSchedule.h b/llvm/include/llvm/MC/MCSchedule.h
index 8c21dbd0ae385..f4c01ce8961ab 100644
--- a/llvm/include/llvm/MC/MCSchedule.h
+++ b/llvm/include/llvm/MC/MCSchedule.h
@@ -317,6 +317,15 @@ struct MCSchedModel {
unsigned MispredictPenalty;
static const unsigned DefaultMispredictPenalty = 10;
+ // StoreLoadForwardingPenalty is the typical number of extra cycles a load
+ // stalls when hardware store-to-load forwarding fails (the load's bytes are
+ // not fully contained in a single, recent store, so it must wait for the
+ // store to drain to cache). A value of 0 means the target does not model
+ // this hazard. Consumed by the SLP vectorizer when a widened store would
+ // break forwarding for a nearby loop-carried load.
+ unsigned StoreLoadForwardingPenalty;
+ static const unsigned DefaultStoreLoadForwardingPenalty = 0;
+
bool PostRAScheduler; // default value is false
bool CompleteModel;
diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td
index c68323389b827..6a1be1f32dedc 100644
--- a/llvm/include/llvm/Target/TargetSchedule.td
+++ b/llvm/include/llvm/Target/TargetSchedule.td
@@ -81,6 +81,9 @@ class SchedMachineModel {
int LoadLatency = -1; // Cycles for loads to access the cache.
int HighLatency = -1; // Approximation of cycles for "high latency" ops.
int MispredictPenalty = -1; // Extra cycles for a mispredicted branch.
+ // Extra cycles a load stalls when store-to-load forwarding fails. 0 (the
+ // default) means the target does not model this hazard.
+ int StoreLoadForwardingPenalty = -1;
// Per-cycle resources tables.
ProcessorItineraries Itineraries = NoItineraries;
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 245cf461b2198..e650917ec2b3f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -175,20 +175,11 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
- /// Returns true if vectorizing the store chain would cause store-to-load
- /// forwarding conflicts due to short loop-carried dependence distances.
- bool hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain, unsigned VF);
-
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
- /// Cached STLF conflict decisions keyed by (first store in chain, VF).
- /// Avoids re-walking the LAA dependence list when the same chain is retried
- /// at multiple vector factors.
- DenseMap<std::pair<const StoreInst *, unsigned>, bool> StlfConflictCache;
-
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 856950ccae595..0a2c14e2af428 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -38,6 +38,12 @@ static cl::opt<unsigned> CacheLineSize(
cl::desc("Use this to override the target cache line size when "
"specified by the user."));
+static cl::opt<int> StoreLoadForwardingConflictCost(
+ "store-load-forwarding-conflict-cost", cl::init(-1), cl::Hidden,
+ cl::desc("Override the per-occurrence cost the SLP vectorizer adds for a "
+ "store-to-load forwarding conflict (negative = use target "
+ "value)."));
+
static cl::opt<unsigned> MinPageSize(
"min-page-size", cl::init(0), cl::Hidden,
cl::desc("Use this to override the target's minimum page size."));
@@ -874,6 +880,15 @@ unsigned TargetTransformInfo::getCacheLineSize() const {
: TTIImpl->getCacheLineSize();
}
+InstructionCost TargetTransformInfo::getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetCostKind CostKind) const {
+ // A negative override means "unset"; fall back to the target's value.
+ if (StoreLoadForwardingConflictCost.getNumOccurrences() > 0 &&
+ StoreLoadForwardingConflictCost >= 0)
+ return InstructionCost(StoreLoadForwardingConflictCost);
+ return TTIImpl->getStoreLoadForwardingConflictCost(VecTy, CostKind);
+}
+
std::optional<unsigned>
TargetTransformInfo::getCacheSize(CacheLevel Level) const {
return TTIImpl->getCacheSize(Level);
diff --git a/llvm/lib/MC/MCSchedule.cpp b/llvm/lib/MC/MCSchedule.cpp
index e2124c1c76355..442e2067bc2e7 100644
--- a/llvm/lib/MC/MCSchedule.cpp
+++ b/llvm/lib/MC/MCSchedule.cpp
@@ -44,6 +44,7 @@ const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth,
DefaultLoadLatency,
DefaultHighLatency,
DefaultMispredictPenalty,
+ DefaultStoreLoadForwardingPenalty,
false,
true,
/*EnableIntervals=*/false,
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index ac4d31de8dbfe..31d640fdd043b 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -53,6 +53,10 @@ def Znver4Model : SchedMachineModel {
// The branch misprediction penalty is in the range from 11 to 18 cycles,
// <...>. The common case penalty is 13 cycles.
let MispredictPenalty = 13;
+ // AMD SOG Zen4, 2.12: a failed store-to-load forward costs roughly a full
+ // L1 load latency of extra delay while the store drains. Used by the SLP
+ // vectorizer to price store chains that would break forwarding.
+ let StoreLoadForwardingPenalty = 13;
let PostRAScheduler = 1; // Enable Post RegAlloc Scheduler pass.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4e128aaba8505..8f146cc1bd87f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -274,7 +274,8 @@ static cl::opt<bool> PerLaneGatherScale(
static cl::opt<bool> EnableSLPStoreLoadForwardCheck(
"slp-store-load-forward-check", cl::init(true), cl::Hidden,
- cl::desc("Check for store-to-load forwarding conflicts in SLP"));
+ cl::desc("Add a cost penalty to store chains whose vectorization would "
+ "break store-to-load forwarding in SLP"));
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
@@ -2136,9 +2137,10 @@ class slpvectorizer::BoUpSLP {
BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
- const DataLayout *DL, OptimizationRemarkEmitter *ORE)
+ const DataLayout *DL, OptimizationRemarkEmitter *ORE,
+ LoopAccessInfoManager *LAIs = nullptr)
: BatchAA(*Aa), F(Func), SE(Se), TTI(Tti), TLI(TLi), LI(Li), DT(Dt),
- AC(AC), DB(DB), DL(DL), ORE(ORE),
+ AC(AC), DB(DB), DL(DL), ORE(ORE), LAIs(LAIs),
Builder(Se->getContext(), TargetFolder(*DL)) {
CodeMetrics::collectEphemeralValues(F, AC, EphValues);
// Use the vector register size specified by the target unless overridden
@@ -2177,6 +2179,13 @@ class slpvectorizer::BoUpSLP {
/// holding live values over call sites.
InstructionCost getSpillCost();
+ /// \returns true if widening the store chain anchored at \p BaseStore into a
+ /// vector store of \p VF elements would break store-to-load forwarding for a
+ /// nearby loop-carried load (a short, misaligned backward dependence). Used
+ /// to add an STLF penalty to the store entry's cost. Result is memoized in
+ /// StlfConflictCache.
+ bool findStoreLoadForwardingConflict(StoreInst *BaseStore, unsigned VF);
+
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
/// final cost.
InstructionCost
@@ -2315,6 +2324,9 @@ class slpvectorizer::BoUpSLP {
TreeEntryToStridedPtrInfoMap.clear();
CurrentLoopNest.clear();
MergedLoopBTCs.clear();
+ // Loads/stores may be reconstructed on the next vectorization attempt, so
+ // stale STLF decisions must not carry over.
+ StlfConflictCache.clear();
}
unsigned getTreeSize() const { return VectorizableTree.size(); }
@@ -6456,6 +6468,14 @@ class slpvectorizer::BoUpSLP {
DemandedBits *DB;
const DataLayout *DL;
OptimizationRemarkEmitter *ORE;
+ /// Cached per-loop memory dependence info, used to detect store-to-load
+ /// forwarding hazards during store-chain costing. May be null.
+ LoopAccessInfoManager *LAIs = nullptr;
+
+ /// Cached STLF conflict decisions keyed by (base store of chain, VF), to
+ /// avoid re-walking the LAA dependence list when the store entry is costed
+ /// repeatedly. Cleared on each buildTree() via deleteTree().
+ SmallDenseMap<std::pair<const StoreInst *, unsigned>, bool> StlfConflictCache;
unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
unsigned MinVecRegSize; // Set by cl::opt (default: 128).
@@ -17742,6 +17762,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
BaseSI->getPointerAddressSpace(), CostKind, OpInfo);
}
}
+ // Widening this store chain can break store-to-load forwarding for a
+ // nearby loop-carried load. Rather than reject the tree outright, add
+ // the target's modeled STLF penalty so a chain that is still profitable
+ // after paying it can vectorize.
+ if (EnableSLPStoreLoadForwardCheck &&
+ findStoreLoadForwardingConflict(BaseSI, E->getVectorFactor()))
+ VecStCost += TTI->getStoreLoadForwardingConflictCost(VecTy, CostKind);
return VecStCost + CommonCost;
};
SmallVector<Value *> PointerOps(VL.size());
@@ -27329,7 +27356,6 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
Stores.clear();
GEPs.clear();
- StlfConflictCache.clear();
bool Changed = false;
// If the target claims to have no vector registers don't attempt
@@ -27348,7 +27374,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
// Use the bottom up slp vectorizer to construct chains that start with
// store instructions.
- BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
+ BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_, LAIs);
// A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
// delete instructions.
@@ -27392,12 +27418,12 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
return Changed;
}
-bool SLPVectorizerPass::hasStoreLoadForwardingConflict(ArrayRef<Value *> Chain,
- unsigned VF) {
- if (Chain.empty() || !LAIs)
+bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
+ unsigned VF) {
+ if (!BaseStore || !LAIs)
return false;
- auto *FirstStore = cast<StoreInst>(Chain[0]);
+ StoreInst *FirstStore = BaseStore;
// Cache lookup: avoid re-walking the LAA dependence list when the same
// chain is retried at multiple vector factors.
@@ -27554,21 +27580,6 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
return false;
}
- // Early bail-out: reject chains that would suffer STLF stalls before doing
- // expensive tree analysis (buildTree, reorder, getTreeCost, etc.).
- if (EnableSLPStoreLoadForwardCheck &&
- hasStoreLoadForwardingConflict(Chain, VF)) {
- LLVM_DEBUG(dbgs() << "SLP: Skipping store chain due to potential "
- "store-to-load forwarding conflict\n");
- using namespace ore;
- R.getORE()->emit(
- OptimizationRemarkMissed(SV_NAME, "StoreLoadForwardConflict",
- cast<StoreInst>(Chain[0]))
- << "Stores not SLP vectorized: short loop-carried dependence "
- "distance could cause store-to-load forwarding stalls");
- return false;
- }
-
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
<< "\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 988ca11ccee6e..bb0447e63f8df 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -23,8 +23,9 @@ target triple = "x86_64-unknown-linux-gnu"
; store at A[i]. The load at A[i-5] is 20 bytes behind:
; 20 % 16 = 4 -> misaligned, load straddles two pending vector stores
; 20 / 16 = 1 -> store still hot in store buffer
-; The check bails out at VF=4. Sub-chains at VF=2 are checked per-base:
-; {A[i+1],A[i+2]} has distance 24, 24%8=0 → safe, so it still vectorizes.
+; The STLF penalty makes the VF=4 store entry unprofitable. Sub-chains at
+; VF=2 are priced per-base: {A[i+1],A[i+2]} has distance 24, 24%8=0 → no
+; conflict, no penalty, so it still vectorizes.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
@@ -418,7 +419,8 @@ for.end:
;
; Both A[i-5] (distance 20 bytes) and A[i-7] (distance 28 bytes) are
; misaligned to a 16-byte vector store and within the safety window. The
-; check fires on the first one encountered and bails out.
+; check fires on the first one encountered and the store entry is penalized,
+; keeping the chain scalar.
;
define void @stlf_multiple_conflicting_loads(ptr noalias %A, i64 %n) {
; STLF-ON-LABEL: define void @stlf_multiple_conflicting_loads(
@@ -593,10 +595,10 @@ for.end:
; Test 9: Motivating example — a[i] = a[i-1] + 1.
;
; Distance = 4 bytes (1 element) from chain base. VectorStoreBytes = 16
-; at VF=4. 4 % 16 = 4 → misaligned → STLF conflict. The full 4-wide
-; chain is rejected. Sub-chains that are individually misaligned are also
-; rejected; only sub-chains where the distance happens to be aligned to
-; the narrower vector store width survive.
+; at VF=4. 4 % 16 = 4 → misaligned → STLF conflict. The full 4-wide store
+; entry is penalized and is not profitable. Sub-chains that are individually
+; misaligned are penalized too; only sub-chains where the distance happens to
+; be aligned to the narrower vector store width vectorize.
;
define void @stlf_conflict_short_backward(ptr noalias %A, i64 %n) {
; STLF-ON-LABEL: define void @stlf_conflict_short_backward(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
new file mode 100644
index 0000000000000..f4657cfe42944
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; Store-to-load forwarding (STLF) is priced into the SLP cost model rather than
+; vetoing vectorization outright. The motivating loop a[i] = a[i-1] + 1 has a
+; misaligned backward dependence that breaks forwarding once the four scalar
+; stores become one 16-byte vector store.
+;
+; With the target's default STLF penalty the small tree is not worth the stall,
+; so it stays (mostly) scalar. Zeroing the penalty via the tuning knob lets the
+; very same tree vectorize to <4 x i32> -- demonstrating this is a cost, not a
+; hard bail-out, so a sufficiently profitable tree can still vectorize.
+;
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefix=PENALTY
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: -store-load-forwarding-conflict-cost=0 \
+; RUN: | FileCheck %s --check-prefix=NOPENALTY
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @stlf_recurrence(ptr noalias %A, i64 %n) {
+; PENALTY-LABEL: define void @stlf_recurrence(
+; PENALTY-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; PENALTY-NEXT: [[ENTRY:.*]]:
+; PENALTY-NEXT: br label %[[FOR_BODY:.*]]
+; PENALTY: [[FOR_BODY]]:
+; PENALTY-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; PENALTY-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; PENALTY-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; PENALTY-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; PENALTY-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; PENALTY-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; PENALTY-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; PENALTY-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; PENALTY-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; PENALTY-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; PENALTY-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; PENALTY-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; PENALTY-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; PENALTY-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; PENALTY-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; PENALTY-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; PENALTY-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; PENALTY-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; PENALTY-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; PENALTY-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; PENALTY: [[FOR_END]]:
+; PENALTY-NEXT: ret void
+;
+; NOPENALTY-LABEL: define void @stlf_recurrence(
+; NOPENALTY-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOPENALTY-NEXT: [[ENTRY:.*]]:
+; NOPENALTY-NEXT: br label %[[FOR_BODY:.*]]
+; NOPENALTY: [[FOR_BODY]]:
+; NOPENALTY-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; NOPENALTY-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; NOPENALTY-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; NOPENALTY-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; NOPENALTY-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; NOPENALTY-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; NOPENALTY-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; NOPENALTY-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; NOPENALTY-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; NOPENALTY-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; NOPENALTY-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; NOPENALTY-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; NOPENALTY: [[FOR_END]]:
+; NOPENALTY-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 1, %entry ], [ %i.next, %for.body ]
+
+ %back.idx = sub i64 %i, 1
+ %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
+ %t = load i32, ptr %back.gep, align 4
+
+ %t1 = add nsw i32 %t, 1
+ %t2 = add nsw i32 %t, 2
+ %t3 = add nsw i32 %t, 3
+ %t4 = add nsw i32 %t, 4
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+
+ %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
+ %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
+ %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
+ %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
+
+ store i32 %t1, ptr %gep0, align 4
+ store i32 %t2, ptr %gep1, align 4
+ store i32 %t3, ptr %gep2, align 4
+ store i32 %t4, ptr %gep3, align 4
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
new file mode 100644
index 0000000000000..701378c3609bf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; A store chain that DOES break store-to-load forwarding, but whose stored
+; values come from a long, isomorphic, vectorizable compute chain over the
+; recurrence load. The value-subtree savings from widening (many scalar ops ->
+; a few vector ops) exceed the modeled STLF stall, so the cost model vectorizes
+; it anyway. This is the case a hard veto would wrongly discard: a veto is the
+; infinite-penalty limit of the cost, and any chain whose vector savings beat
+; the stall should still vectorize.
+;
+; Default run: the znver5 penalty (StoreLoadForwardingPenalty = 13) is paid but
+; the whole chain still vectorizes to <4 x i32> because the value tree saves
+; more than that. The absurd-penalty run (the veto/infinite limit) prices the
+; chain out again, proving the store entry really is charged the STLF penalty
+; (the chain is a genuine forwarding conflict, not a no-op).
+;
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefix=VEC
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: -store-load-forwarding-conflict-cost=1000 \
+; RUN: | FileCheck %s --check-prefix=VETO
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @stlf_profitable(ptr noalias %A, i64 %n) {
+; VEC-LABEL: define void @stlf_profitable(
+; VEC-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[FOR_BODY:.*]]
+; VEC: [[FOR_BODY]]:
+; VEC-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC-NEXT: [[BI:%.*]] = add i64 [[I]], -1
+; VEC-NEXT: [[BGEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BI]]
+; VEC-NEXT: [[T:%.*]] = load i32, ptr [[BGEP]], align 4
+; VEC-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; VEC-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i32 0
+; VEC-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; VEC-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; VEC-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
+; VEC-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP3]], <i32 7, i32 8, i32 9, i32 10>
+; VEC-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP4]], [[TMP2]]
+; VEC-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], <i32 11, i32 12, i32 13, i32 14>
+; VEC-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], [[TMP4]]
+; VEC-NEXT: store <4 x i32> [[TMP7]], ptr [[G0]], align 4
+; VEC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; VEC-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; VEC-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; VEC: [[FOR_END]]:
+; VEC-NEXT: ret void
+;
+; VETO-LABEL: define void @stlf_profitable(
+; VETO-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VETO-NEXT: [[ENTRY:.*]]:
+; VETO-NEXT: br label %[[FOR_BODY:.*]]
+; VETO: [[FOR_BODY]]:
+; VETO-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; VETO-NEXT: [[BI:%.*]] = add i64 [[I]], -1
+; VETO-NEXT: [[BGEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BI]]
+; VETO-NEXT: [[T:%.*]] = load i32, ptr [[BGEP]], align 4
+; VETO-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i32 0
+; VETO-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; VETO-NEXT: [[TMP2:%.*]] = add <2 x i32> [[TMP1]], <i32 1, i32 4>
+; VETO-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[TMP2]], [[TMP2]]
+; VETO-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP3]], <i32 7, i32 10>
+; VETO-NEXT: [[TMP5:%.*]] = mul <2 x i32> [[TMP4]], [[TMP2]]
+; VETO-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], <i32 11, i32 14>
+; VETO-NEXT: [[TMP7:%.*]] = mul <2 x i32> [[TMP6]], [[TMP4]]
+; VETO-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; VETO-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; VETO-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; VETO-NEXT: [[G1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; VETO-NEXT: [[G3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; VETO-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i32 0
+; VETO-NEXT: store i32 [[TMP8]], ptr [[G0]], align 4
+; VETO-NEXT: [[TMP9:%.*]] = add <2 x i32> [[TMP1]], <i32 2, i32 3>
+; VETO-NEXT: [[TMP10:%.*]] = mul <2 x i32> [[TMP9]], [[TMP9]]
+; VETO-NEXT: [[TMP11:%.*]] = add <2 x i32> [[TMP10]], <i32 8, i32 9>
+; VETO-NEXT: [[TMP12:%.*]] = mul <2 x i32> [[TMP11]], [[TMP9]]
+; VETO-NEXT: [[TMP13:%.*]] = add <2 x i32> [[TMP12]], <i32 12, i32 13>
+; VETO-NEXT: [[TMP14:%.*]] = mul <2 x i32> [[TMP13]], [[TMP11]]
+; VETO-NEXT: store <2 x i32> [[TMP14]], ptr [[G1]], align 4
+; VETO-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP7]], i32 1
+; VETO-NEXT: store i32 [[TMP15]], ptr [[G3]], align 4
+; VETO-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; VETO-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; VETO-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; VETO: [[FOR_END]]:
+; VETO-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 1, %entry ], [ %i.next, %for.body ]
+ %bi = add i64 %i, -1
+ %bgep = getelementptr inbounds i32, ptr %A, i64 %bi
+ %t = load i32, ptr %bgep, align 4
+
+ %a0 = add i32 %t, 1
+ %b0 = mul i32 %a0, %a0
+ %c0 = add i32 %b0, 7
+ %d0 = mul i32 %c0, %a0
+ %e0 = add i32 %d0, 11
+ %f0 = mul i32 %e0, %c0
+
+ %a1 = add i32 %t, 2
+ %b1 = mul i32 %a1, %a1
+ %c1 = add i32 %b1, 8
+ %d1 = mul i32 %c1, %a1
+ %e1 = add i32 %d1, 12
+ %f1 = mul i32 %e1, %c1
+
+ %a2 = add i32 %t, 3
+ %b2 = mul i32 %a2, %a2
+ %c2 = add i32 %b2, 9
+ %d2 = mul i32 %c2, %a2
+ %e2 = add i32 %d2, 13
+ %f2 = mul i32 %e2, %c2
+
+ %a3 = add i32 %t, 4
+ %b3 = mul i32 %a3, %a3
+ %c3 = add i32 %b3, 10
+ %d3 = mul i32 %c3, %a3
+ %e3 = add i32 %d3, 14
+ %f3 = mul i32 %e3, %c3
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+ %g0 = getelementptr inbounds i32, ptr %A, i64 %i
+ %g1 = getelementptr inbounds i32, ptr %A, i64 %i1
+ %g2 = getelementptr inbounds i32, ptr %A, i64 %i2
+ %g3 = getelementptr inbounds i32, ptr %A, i64 %i3
+ store i32 %f0, ptr %g0, align 4
+ store i32 %f1, ptr %g1, align 4
+ store i32 %f2, ptr %g2, align 4
+ store i32 %f3, ptr %g3, align 4
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index c3a9d69d6ec3e..b68b7f86096f6 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1456,6 +1456,7 @@ void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
emitProcessorProp(OS, PM.ModelDef, "LoadLatency", ',');
emitProcessorProp(OS, PM.ModelDef, "HighLatency", ',');
emitProcessorProp(OS, PM.ModelDef, "MispredictPenalty", ',');
+ emitProcessorProp(OS, PM.ModelDef, "StoreLoadForwardingPenalty", ',');
bool PostRAScheduler =
(PM.ModelDef ? PM.ModelDef->getValueAsBit("PostRAScheduler") : false);
More information about the llvm-commits
mailing list