[llvm] [SLP] Bail out on store-to-load forwarding hazards (PR #199606)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 22:47:32 PDT 2026
https://github.com/mbhade-amd updated https://github.com/llvm/llvm-project/pull/199606
>From b612a00c90ccd0c1609e42224c8aa16c39d1b174 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Fri, 31 Jul 2026 15:10:14 +0530
Subject: [PATCH 1/4] [SLP] Model store-to-load forwarding hazards as a cost
penalty
Widening a consecutive store chain can break store-to-load forwarding
(STLF) for a nearby loop-carried load whose backward distance is
misaligned to the vector store width and short enough that the store is
still in the store buffer, causing a pipeline stall.
Add a target cost hook, getStoreLoadForwardingConflictCost, backed by a
new MCSchedule StoreLoadForwardingConflict penalty (defaulting to 10
cycles, set for znver4). When such a conflict is detected, charge this
penalty to the store entry in the SLP cost model instead of rejecting
the tree outright, so a chain that remains profitable after paying it
can still vectorize.
Conflicting loads are enumerated from the SLP tree via a def-use walk
seeded from the tree scalars and following operands (staying inside the
loop, reaching loads below gather/splat leaves), rather than scanning
the whole loop or LAA's dependence list. The conflict predicate is
shared with LoopAccessAnalysis
(MemoryDepChecker::isStoreLoadForwardingConflict).
---
.../llvm/Analysis/LoopAccessAnalysis.h | 19 +
.../llvm/Analysis/TargetTransformInfo.h | 9 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 9 +
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 7 +
llvm/include/llvm/MC/MCSchedule.h | 11 +
llvm/include/llvm/Target/TargetSchedule.td | 5 +
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 1 +
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 11 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 14 +
llvm/lib/MC/MCSchedule.cpp | 1 +
llvm/lib/Target/X86/X86ScheduleZnver4.td | 4 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 131 ++++++
.../X86/store-load-forward-conflict.ll | 401 ++++++++++--------
.../X86/store-load-forward-cost.ll | 105 +++++
.../X86/store-load-forward-profitable.ll | 145 +++++++
llvm/utils/TableGen/SubtargetEmitter.cpp | 1 +
16 files changed, 683 insertions(+), 191 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 392321448c895..dbd0248f2e58f 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -235,6 +235,25 @@ class MemoryDepChecker {
std::numeric_limits<uint64_t>::max();
}
+ /// Returns true if a memory dependence at byte distance \p Distance between
+ /// a store and load (both with element size \p TypeByteSize bytes) would
+ /// prevent store-to-load forwarding when the store is widened to
+ /// \p VectorStoreSize bytes.
+ ///
+ /// The predicate fires when (a) the load is misaligned w.r.t. the widened
+ /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
+ /// (b) the conflicting store is still likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
+ /// Both couldPreventStoreLoadForward and SLPVectorizer use this as their
+ /// core STLF cost-model gate.
+ static bool isStoreLoadForwardingConflict(uint64_t Distance,
+ uint64_t VectorStoreSize,
+ uint64_t TypeByteSize) {
+ const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
+ return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
+ Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ }
+
/// Return safe power-of-2 number of elements, which do not prevent store-load
/// forwarding, multiplied by the size of the elements in bits.
uint64_t getStoreLoadForwardSafeDistanceInBits() const {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..647ae38de8b72 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1415,6 +1415,15 @@ class TargetTransformInfo {
/// \return The size of a cache line in bytes.
LLVM_ABI unsigned getCacheLineSize() const;
+ /// \return The penalty for a store-to-load forwarding (STLF) hazard when a
+ /// chain of stores is widened into the vector store \p VecTy inside a loop
+ /// and a nearby loop-carried load straddles that widened store, so hardware
+ /// store-to-load forwarding fails and the load stalls until the store drains
+ /// to L1. Targets return the modeled per-occurrence penalty, or 0 if they do
+ /// not model this hazard.
+ LLVM_ABI InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetCostKind CostKind) const;
+
/// The possible cache levels
enum class CacheLevel {
L1D, // The L1 data cache
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 25574a2d2c0e5..2d402429a8b8a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -23,6 +23,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/MC/MCSchedule.h"
#include <optional>
#include <utility>
@@ -680,6 +681,14 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getCacheLineSize() const { return 0; }
+
+ virtual InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
+ // No subtarget scheduling model is available here, so fall back to the
+ // conservative default STLF stall penalty (targets with a real model go
+ // through BasicTTIImpl and use their own value instead).
+ return InstructionCost(MCSchedModel::DefaultStoreLoadForwardingPenalty);
+ }
virtual std::optional<unsigned>
getCacheSize(TargetTransformInfo::CacheLevel Level) const {
switch (Level) {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..eb0d09f634a44 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -886,6 +886,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return getST()->getCacheLineSize();
}
+ InstructionCost getStoreLoadForwardingConflictCost(
+ Type *VecTy, TTI::TargetCostKind CostKind) const override {
+ // The per-occurrence STLF stall penalty is a microarchitectural cycle
+ // count sourced from the subtarget's scheduling model (0 = not modeled).
+ return InstructionCost(getST()->getSchedModel().StoreLoadForwardingPenalty);
+ }
+
unsigned getPrefetchDistance() const override {
return getST()->getPrefetchDistance();
}
diff --git a/llvm/include/llvm/MC/MCSchedule.h b/llvm/include/llvm/MC/MCSchedule.h
index dd65284671d71..a065123594bb0 100644
--- a/llvm/include/llvm/MC/MCSchedule.h
+++ b/llvm/include/llvm/MC/MCSchedule.h
@@ -326,6 +326,17 @@ struct MCSchedModel {
unsigned MispredictPenalty;
static constexpr unsigned DefaultMispredictPenalty = 10;
+ // StoreLoadForwardingPenalty is the typical number of extra cycles a load
+ // stalls when hardware store-to-load forwarding fails (the load's bytes are
+ // not fully contained in a single, recent store, so it must wait for the
+ // store to drain to cache). Consumed by the SLP vectorizer when a widened
+ // store would break forwarding for a nearby loop-carried load.
+ // The failed-forwarding stall is roughly 10-15 cycles on current cores, so
+ // the default uses a conservative 10 cycles for targets that do not override
+ // it; individual targets may set a more precise value (e.g. Zen4 uses 13).
+ unsigned StoreLoadForwardingPenalty;
+ static const unsigned DefaultStoreLoadForwardingPenalty = 10;
+
bool PostRAScheduler; // default value is false
bool CompleteModel;
diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td
index c68323389b827..5f23afb2131d7 100644
--- a/llvm/include/llvm/Target/TargetSchedule.td
+++ b/llvm/include/llvm/Target/TargetSchedule.td
@@ -81,6 +81,11 @@ class SchedMachineModel {
int LoadLatency = -1; // Cycles for loads to access the cache.
int HighLatency = -1; // Approximation of cycles for "high latency" ops.
int MispredictPenalty = -1; // Extra cycles for a mispredicted branch.
+ // Extra cycles a load stalls when store-to-load forwarding fails. Left unset
+ // (-1), targets fall back to a conservative default (see
+ // MCSchedModel::DefaultStoreLoadForwardingPenalty); a target may override
+ // with a more precise value.
+ int StoreLoadForwardingPenalty = -1;
// Per-cycle resources tables.
ProcessorItineraries Itineraries = NoItineraries;
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 538d62626b37b..b35a49c227487 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -19,6 +19,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 1f13106214ea9..f5e57ea7c4d2a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1924,20 +1924,17 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance,
// place. Vectorizing in such cases does not make sense.
// Store-load forwarding distance.
- // After this many iterations store-to-load forwarding conflicts should not
- // cause any slowdowns.
- const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
// Maximum vector factor.
uint64_t MaxVFWithoutSLForwardIssuesPowerOf2 =
std::min(VectorizerParams::MaxVectorWidth * TypeByteSize,
MaxStoreLoadForwardSafeDistanceInBits);
- // Compute the smallest VF at which the store and load would be misaligned.
+ // Compute the smallest VF at which the store and load would be misaligned
+ // and recent enough to still be in the store buffer. The predicate is
+ // shared with SLP via MemoryDepChecker::isStoreLoadForwardingConflict.
for (uint64_t VF = 2 * TypeByteSize;
VF <= MaxVFWithoutSLForwardIssuesPowerOf2; VF *= 2) {
- // If the number of vector iteration between the store and the load are
- // small we could incur conflicts.
- if (Distance % VF && Distance / VF < NumItersForStoreLoadThroughMemory) {
+ if (isStoreLoadForwardingConflict(Distance, VF, TypeByteSize)) {
MaxVFWithoutSLForwardIssuesPowerOf2 = (VF >> 1);
break;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 595c2d10dd118..b4a57d1c00635 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -38,6 +38,12 @@ static cl::opt<unsigned> CacheLineSize(
cl::desc("Use this to override the target cache line size when "
"specified by the user."));
+static cl::opt<int> StoreLoadForwardingConflictCost(
+ "store-load-forwarding-conflict-cost", cl::init(-1), cl::Hidden,
+ cl::desc("Override the per-occurrence cost the SLP vectorizer adds for a "
+ "store-to-load forwarding conflict (negative = use target "
+ "value)."));
+
static cl::opt<unsigned> MinPageSize(
"min-page-size", cl::init(0), cl::Hidden,
cl::desc("Use this to override the target's minimum page size."));
@@ -881,6 +887,14 @@ unsigned TargetTransformInfo::getCacheLineSize() const {
: TTIImpl->getCacheLineSize();
}
+InstructionCost TargetTransformInfo::getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetCostKind CostKind) const {
+ // A negative override means "unset"; fall back to the target's value.
+ if (StoreLoadForwardingConflictCost >= 0)
+ return InstructionCost(StoreLoadForwardingConflictCost);
+ return TTIImpl->getStoreLoadForwardingConflictCost(VecTy, CostKind);
+}
+
std::optional<unsigned>
TargetTransformInfo::getCacheSize(CacheLevel Level) const {
return TTIImpl->getCacheSize(Level);
diff --git a/llvm/lib/MC/MCSchedule.cpp b/llvm/lib/MC/MCSchedule.cpp
index e2124c1c76355..442e2067bc2e7 100644
--- a/llvm/lib/MC/MCSchedule.cpp
+++ b/llvm/lib/MC/MCSchedule.cpp
@@ -44,6 +44,7 @@ const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth,
DefaultLoadLatency,
DefaultHighLatency,
DefaultMispredictPenalty,
+ DefaultStoreLoadForwardingPenalty,
false,
true,
/*EnableIntervals=*/false,
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index ac4d31de8dbfe..31d640fdd043b 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -53,6 +53,10 @@ def Znver4Model : SchedMachineModel {
// The branch misprediction penalty is in the range from 11 to 18 cycles,
// <...>. The common case penalty is 13 cycles.
let MispredictPenalty = 13;
+ // AMD SOG Zen4, 2.12: a failed store-to-load forward costs roughly a full
+ // L1 load latency of extra delay while the store drains. Used by the SLP
+ // vectorizer to price store chains that would break forwarding.
+ let StoreLoadForwardingPenalty = 13;
let PostRAScheduler = 1; // Enable Post RegAlloc Scheduler pass.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 069facf6fb4e0..fe5c1b7338e58 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -329,6 +329,11 @@ static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
"guarded scalar region cost, before versioning is rejected to "
"avoid pessimizing the scalar fallback path."));
+static cl::opt<bool> EnableSLPStoreLoadForwardCheck(
+ "slp-store-load-forward-check", cl::init(true), cl::Hidden,
+ cl::desc("Add a cost penalty to store chains whose vectorization would "
+ "break store-to-load forwarding in SLP"));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -1005,6 +1010,13 @@ class slpvectorizer::BoUpSLP {
/// holding live values over call sites.
InstructionCost getSpillCost();
+ /// \returns true if widening the store chain anchored at \p BaseStore into a
+ /// vector store of \p VF elements would break store-to-load forwarding for a
+ /// nearby loop-carried load (a short, misaligned backward dependence). Used
+ /// to add an STLF penalty to the store entry's cost. Result is memoized in
+ /// StlfConflictCache.
+ bool findStoreLoadForwardingConflict(StoreInst *BaseStore, unsigned VF);
+
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
/// final cost.
InstructionCost
@@ -1240,6 +1252,9 @@ class slpvectorizer::BoUpSLP {
TreeEntryToStridedPtrInfoMap.clear();
CurrentLoopNest.clear();
MergedLoopBTCs.clear();
+ // Loads/stores may be reconstructed on the next vectorization attempt, so
+ // stale STLF decisions must not carry over.
+ StlfConflictCache.clear();
}
unsigned getTreeSize() const { return VectorizableTree.size(); }
@@ -5685,6 +5700,11 @@ class slpvectorizer::BoUpSLP {
const DataLayout *DL;
OptimizationRemarkEmitter *ORE;
+ /// Cached STLF conflict decisions keyed by (base store of chain, VF), to
+ /// avoid re-walking the tree when the store entry is costed repeatedly.
+ /// Cleared on each buildTree() via deleteTree().
+ SmallDenseMap<std::pair<const StoreInst *, unsigned>, bool> StlfConflictCache;
+
unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
unsigned MinVecRegSize; // Set by cl::opt (default: 128).
@@ -17721,6 +17741,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
BaseSI->getPointerAddressSpace(), CostKind, OpInfo);
}
}
+ // Widening this store chain can break store-to-load forwarding for a
+ // nearby loop-carried load. Rather than reject the tree outright, add
+ // the target's modeled STLF penalty so a chain that is still profitable
+ // after paying it can vectorize.
+ if (EnableSLPStoreLoadForwardCheck && E->State == TreeEntry::Vectorize &&
+ !E->getInterleaveFactor() &&
+ findStoreLoadForwardingConflict(BaseSI, E->getVectorFactor()))
+ VecStCost += TTI->getStoreLoadForwardingConflictCost(VecTy, CostKind);
return VecStCost + CommonCost;
};
SmallVector<Value *> PointerOps(VL.size());
@@ -28257,6 +28285,109 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
return Changed;
}
+bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
+ unsigned VF) {
+ if (!BaseStore)
+ return false;
+
+ StoreInst *FirstStore = BaseStore;
+
+ // Memoize per (store, VF); the entry is re-costed repeatedly.
+ auto Key = std::make_pair(FirstStore, VF);
+ auto CacheIt = StlfConflictCache.find(Key);
+ if (CacheIt != StlfConflictCache.end())
+ return CacheIt->second;
+
+ auto CacheAndReturn = [&](bool Result) -> bool {
+ StlfConflictCache[Key] = Result;
+ // No conflict at VF implies none at any smaller power-of-2 VF, so seed
+ // those entries too. Conflicts do not propagate downward.
+ if (!Result && isPowerOf2_32(VF))
+ for (unsigned V = VF / 2; V >= 2; V /= 2)
+ StlfConflictCache.try_emplace(std::make_pair(FirstStore, V), false);
+ return Result;
+ };
+
+ Loop *L = LI->getLoopFor(FirstStore->getParent());
+ if (!L)
+ return CacheAndReturn(false);
+
+ Type *ValueTy = FirstStore->getValueOperand()->getType();
+ TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
+ if (StoreSize.isScalable())
+ return CacheAndReturn(false);
+ uint64_t ElementSize = StoreSize.getFixedValue();
+ if (ElementSize == 0)
+ return CacheAndReturn(false);
+ uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
+ << " ElementSize=" << ElementSize
+ << " VectorStoreBytes=" << VectorStoreBytes << "\n");
+
+ // Enumerate candidate loads from the tree, not the whole loop. A conflicting
+ // load feeds the vectorized nodes, so it is reachable by walking operands
+ // from the tree scalars (including through gather leaves such as splats).
+ // Stay inside the loop to bound the walk by the tree's cone.
+ Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
+ SmallPtrSet<LoadInst *, 8> CandidateLoads;
+ SmallPtrSet<const Value *, 32> Visited;
+ SmallVector<Value *, 32> Worklist;
+ for (const std::unique_ptr<TreeEntry> &TEPtr : VectorizableTree) {
+ const TreeEntry *TE = TEPtr.get();
+ if (DeletedNodes.contains(TE))
+ continue;
+ Worklist.append(TE->Scalars.begin(), TE->Scalars.end());
+ }
+ while (!Worklist.empty()) {
+ Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ if (auto *LoadI = dyn_cast<LoadInst>(V)) {
+ if (LoadI->isSimple() &&
+ getUnderlyingObject(LoadI->getPointerOperand()) == StoreBase)
+ CandidateLoads.insert(LoadI);
+ continue;
+ }
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || !L->contains(I))
+ continue;
+ Worklist.append(I->op_begin(), I->op_end());
+ }
+
+ if (CandidateLoads.empty())
+ return CacheAndReturn(false);
+
+ // For each candidate load, the widened chain becomes one wide store at the
+ // base; check whether the load straddles two such wide stores.
+ for (LoadInst *LoadI : CandidateLoads) {
+ std::optional<int64_t> Diff =
+ getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
+ LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
+ /*StrictCheck=*/true, /*CheckType=*/false);
+ if (!Diff || *Diff >= 0)
+ continue;
+
+ uint64_t Distance = -static_cast<uint64_t>(*Diff) * ElementSize;
+ LLVM_DEBUG(dbgs() << "SLP: STLF: load=" << *LoadI << " distance="
+ << Distance << " bytes from chain base\n");
+
+ // Conflict if the load is misaligned to the wide store within the recency
+ // window.
+ if (MemoryDepChecker::isStoreLoadForwardingConflict(
+ Distance, VectorStoreBytes, ElementSize)) {
+ LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: "
+ << (isVectorized(LoadI) ? "widened" : "scalar")
+ << " load, distance " << Distance
+ << " bytes, vector store width " << VectorStoreBytes
+ << " bytes, misalignment "
+ << (Distance % VectorStoreBytes) << "\n");
+ return CacheAndReturn(true);
+ }
+ }
+
+ return CacheAndReturn(false);
+}
+
std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index b705f5cea5347..906c560635fc0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -1,5 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-ON
+; RUN: opt < %s -passes=slp-vectorizer -slp-store-load-forward-check=false -S \
+; RUN: -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-OFF
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
;
; Test 1: Backward loop-carried load at misaligned distance.
@@ -16,31 +23,58 @@
; store at A[i]. The load at A[i-5] is 20 bytes behind:
; 20 % 16 = 4 -> misaligned, load straddles two pending vector stores
; 20 / 16 = 1 -> store still hot in store buffer
-; The new check should bail out and leave the four-wide chain scalar.
+; The STLF penalty makes the VF=4 store entry unprofitable. Sub-chains at
+; VF=2 are priced per-base: {A[i+1],A[i+2]} has distance 24, 24%8=0 → no
+; conflict, no penalty, so it still vectorizes.
;
define void @stlf_conflict_backward_misaligned(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i64 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -92,9 +126,8 @@ for.end:
; The check should NOT fire; vectorization is allowed.
;
define void @stlf_no_conflict_backward_aligned(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_backward_aligned(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
@@ -157,7 +190,6 @@ for.end:
; Vectorization should proceed.
;
define void @stlf_no_conflict_forward(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_forward(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -222,7 +254,6 @@ for.end:
; store buffer is hot. Outside loops the check should not fire.
;
define void @stlf_no_conflict_no_loop(ptr noalias %A) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_no_loop(
; CHECK-SAME: ptr noalias [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 -5
@@ -260,7 +291,6 @@ define void @stlf_no_conflict_no_loop(ptr noalias %A) {
; widened store, so the check should not fire.
;
define void @stlf_no_conflict_different_base(ptr noalias %A, ptr noalias %B,
-;
; CHECK-LABEL: define void @stlf_no_conflict_different_base(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -327,7 +357,6 @@ for.end:
; finds no offending load and vectorization proceeds.
;
define void @stlf_volatile_load_skipped(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_volatile_load_skipped(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -390,35 +419,67 @@ for.end:
;
; Both A[i-5] (distance 20 bytes) and A[i-7] (distance 28 bytes) are
; misaligned to a 16-byte vector store and within the safety window. The
-; check fires on the first one encountered and bails out.
+; check fires on the first one encountered and the store entry is penalized,
+; keeping the chain scalar.
;
define void @stlf_multiple_conflicting_loads(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_multiple_conflicting_loads(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
-; CHECK-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
-; CHECK-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
-; CHECK-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
-; CHECK-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
-; CHECK-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-ON-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-ON-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-ON-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-ON-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-ON-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[SUM]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i32 [[SUM]], 2
+; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[SUM]], 3
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[SUM]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I2:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I2]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: store i32 [[T2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP2]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_multiple_conflicting_loads(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; STLF-OFF-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; STLF-OFF-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; STLF-OFF-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; STLF-OFF-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; STLF-OFF-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -471,7 +532,6 @@ for.end:
; conflict, and vectorization proceeds.
;
define void @stlf_pointer_diff_unknown(ptr noalias %A, ptr noalias %IdxA, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_pointer_diff_unknown(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[IDXA:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -532,35 +592,62 @@ for.end:
}
;
-; Test 9: Distance < VectorStoreBytes (load fully within previous vector
-; store).
-;
-; The backward load is only 1 element (4 bytes) before the stores; the entire
-; load lies inside the previous iteration's 16-byte vector store, so STLF
-; can forward. The check's `Distance < VectorStoreBytes` early-continue path
-; catches this and allows vectorization.
-;
-define void @stlf_no_conflict_within_prev_store(ptr noalias %A, i64 %n) {
-;
-; CHECK-LABEL: define void @stlf_no_conflict_within_prev_store(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; Test 9: Motivating example — a[i] = a[i-1] + 1.
+;
+; Distance = 4 bytes (1 element) from chain base. VectorStoreBytes = 16
+; at VF=4. 4 % 16 = 4 → misaligned → STLF conflict. The full 4-wide store
+; entry is penalized and is not profitable. Sub-chains that are individually
+; misaligned are penalized too; only sub-chains where the distance happens to
+; be aligned to the narrower vector store width vectorize.
+;
+define void @stlf_conflict_short_backward(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @stlf_conflict_short_backward(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i64 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_short_backward(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
@@ -600,79 +687,7 @@ for.end:
}
;
-; Test 10: Loop body too large -> STLF check skipped.
-;
-; Same misaligned-distance shape as Test 1, but the third RUN line lowers
-; the SLP loop-size threshold to 10. The loop body is ~20 instructions
-; (well above 10 yet well below the default 128). With the default
-; threshold of 128 the check still runs and rejects vectorization, with
-; the STLF flag forced off vectorization is allowed, and with the
-; threshold lowered to 10 the loop is treated as large so the check is
-; skipped and vectorization is allowed.
-;
-define void @stlf_large_loop_bypassed(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_large_loop_bypassed(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %for.body
-
-for.body:
- %i = phi i64 [ 5, %entry ], [ %i.next, %for.body ]
-
- %back.idx = sub i64 %i, 5
- %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
- %t = load i32, ptr %back.gep, align 4
-
- %t1 = add nsw i32 %t, 1
- %t2 = add nsw i32 %t, 2
- %t3 = add nsw i32 %t, 3
- %t4 = add nsw i32 %t, 4
-
- %i1 = add nuw nsw i64 %i, 1
- %i2 = add nuw nsw i64 %i, 2
- %i3 = add nuw nsw i64 %i, 3
-
- %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
- %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
- %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
- %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
-
- store i32 %t1, ptr %gep0, align 4
- store i32 %t2, ptr %gep1, align 4
- store i32 %t3, ptr %gep2, align 4
- store i32 %t4, ptr %gep3, align 4
-
- %i.next = add nuw nsw i64 %i, 4
- %cmp = icmp slt i64 %i.next, %n
- br i1 %cmp, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
-;
-; Test 11: Distance just past NumItersForSafety boundary.
+; Test 10: Distance just past NumItersForSafety boundary.
;
; For i32 with VF=4: ElementSize=4, VectorStoreBytes=16,
; NumItersForSafety = 8 * ElementSize = 32. The check fires only when
@@ -682,7 +697,6 @@ for.end:
; (misaligned). Vectorization proceeds.
;
define void @stlf_no_conflict_iter_safety_boundary(ptr noalias %A, i64 %n) {
-;
; CHECK-LABEL: define void @stlf_no_conflict_iter_safety_boundary(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -741,7 +755,7 @@ for.end:
}
;
-; Test 12: Non-i32 element type (i64 VF=2).
+; Test 11: Non-i32 element type (i64 VF=2).
;
; Verifies the byte-distance math works for wider types. With i64
; (ElementSize=8) and VF=2: VectorStoreBytes=16, NumItersForSafety=64.
@@ -749,28 +763,47 @@ for.end:
; 24%16=8 != 0, so the check fires.
;
define void @stlf_conflict_backward_misaligned_i64(ptr noalias %A, i64 %n) {
-;
-;
-;
-; CHECK-LABEL: define void @stlf_conflict_backward_misaligned_i64(
-; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
-; CHECK-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
-; CHECK-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
-; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
-; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret void
+; STLF-ON-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i64 [[T]], 1
+; STLF-ON-NEXT: [[T2:%.*]] = add nsw i64 [[T]], 2
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: store i64 [[T1]], ptr [[GEP0]], align 8
+; STLF-ON-NEXT: store i64 [[T2]], ptr [[GEP1]], align 8
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_backward_misaligned_i64(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 3, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 3
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i64, ptr [[BACK_GEP]], align 8
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[T]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <2 x i64> [[TMP1]], <i64 1, i64 2>
+; STLF-OFF-NEXT: store <2 x i64> [[TMP2]], ptr [[GEP0]], align 8
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
new file mode 100644
index 0000000000000..d371948c3eb6b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-cost.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; Store-to-load forwarding (STLF) is priced into the SLP cost model rather than
+; vetoing vectorization outright. The motivating loop a[i] = a[i-1] + 1 has a
+; misaligned backward dependence that breaks forwarding once the four scalar
+; stores become one 16-byte vector store.
+;
+; With the target's default STLF penalty the small tree is not worth the stall,
+; so it stays (mostly) scalar. Zeroing the penalty via the tuning knob lets the
+; very same tree vectorize to <4 x i32> -- demonstrating this is a cost, not a
+; hard bail-out, so a sufficiently profitable tree can still vectorize.
+;
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefix=PENALTY
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: -store-load-forwarding-conflict-cost=0 \
+; RUN: | FileCheck %s --check-prefix=NOPENALTY
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @stlf_recurrence(ptr noalias %A, i64 %n) {
+; PENALTY-LABEL: define void @stlf_recurrence(
+; PENALTY-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; PENALTY-NEXT: [[ENTRY:.*]]:
+; PENALTY-NEXT: br label %[[FOR_BODY:.*]]
+; PENALTY: [[FOR_BODY]]:
+; PENALTY-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; PENALTY-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; PENALTY-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; PENALTY-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; PENALTY-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; PENALTY-NEXT: [[T4:%.*]] = add nsw i32 [[T]], 4
+; PENALTY-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; PENALTY-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; PENALTY-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; PENALTY-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; PENALTY-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; PENALTY-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; PENALTY-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i64 0
+; PENALTY-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; PENALTY-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; PENALTY-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; PENALTY-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
+; PENALTY-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; PENALTY-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; PENALTY-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; PENALTY: [[FOR_END]]:
+; PENALTY-NEXT: ret void
+;
+; NOPENALTY-LABEL: define void @stlf_recurrence(
+; NOPENALTY-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOPENALTY-NEXT: [[ENTRY:.*]]:
+; NOPENALTY-NEXT: br label %[[FOR_BODY:.*]]
+; NOPENALTY: [[FOR_BODY]]:
+; NOPENALTY-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; NOPENALTY-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 1
+; NOPENALTY-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; NOPENALTY-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; NOPENALTY-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; NOPENALTY-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
+; NOPENALTY-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; NOPENALTY-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; NOPENALTY-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; NOPENALTY-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; NOPENALTY-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; NOPENALTY-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; NOPENALTY: [[FOR_END]]:
+; NOPENALTY-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 1, %entry ], [ %i.next, %for.body ]
+
+ %back.idx = sub i64 %i, 1
+ %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
+ %t = load i32, ptr %back.gep, align 4
+
+ %t1 = add nsw i32 %t, 1
+ %t2 = add nsw i32 %t, 2
+ %t3 = add nsw i32 %t, 3
+ %t4 = add nsw i32 %t, 4
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+
+ %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
+ %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
+ %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
+ %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
+
+ store i32 %t1, ptr %gep0, align 4
+ store i32 %t2, ptr %gep1, align 4
+ store i32 %t3, ptr %gep2, align 4
+ store i32 %t4, ptr %gep3, align 4
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
new file mode 100644
index 0000000000000..4b22025a1f4bd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-profitable.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; A store chain that DOES break store-to-load forwarding, but whose stored
+; values come from a long, isomorphic, vectorizable compute chain over the
+; recurrence load. The value-subtree savings from widening (many scalar ops ->
+; a few vector ops) exceed the modeled STLF stall, so the cost model vectorizes
+; it anyway. This is the case a hard veto would wrongly discard: a veto is the
+; infinite-penalty limit of the cost, and any chain whose vector savings beat
+; the stall should still vectorize.
+;
+; Default run: the znver5 penalty (StoreLoadForwardingPenalty = 13) is paid but
+; the whole chain still vectorizes to <4 x i32> because the value tree saves
+; more than that. The absurd-penalty run (the veto/infinite limit) prices the
+; chain out again, proving the store entry really is charged the STLF penalty
+; (the chain is a genuine forwarding conflict, not a no-op).
+;
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefix=VEC
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: -store-load-forwarding-conflict-cost=1000 \
+; RUN: | FileCheck %s --check-prefix=VETO
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @stlf_profitable(ptr noalias %A, i64 %n) {
+; VEC-LABEL: define void @stlf_profitable(
+; VEC-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: br label %[[FOR_BODY:.*]]
+; VEC: [[FOR_BODY]]:
+; VEC-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; VEC-NEXT: [[BI:%.*]] = add i64 [[I]], -1
+; VEC-NEXT: [[BGEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BI]]
+; VEC-NEXT: [[T:%.*]] = load i32, ptr [[BGEP]], align 4
+; VEC-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; VEC-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
+; VEC-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; VEC-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; VEC-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
+; VEC-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP3]], <i32 7, i32 8, i32 9, i32 10>
+; VEC-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP4]], [[TMP2]]
+; VEC-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], <i32 11, i32 12, i32 13, i32 14>
+; VEC-NEXT: [[TMP7:%.*]] = mul <4 x i32> [[TMP6]], [[TMP4]]
+; VEC-NEXT: store <4 x i32> [[TMP7]], ptr [[G0]], align 4
+; VEC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; VEC-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; VEC-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; VEC: [[FOR_END]]:
+; VEC-NEXT: ret void
+;
+; VETO-LABEL: define void @stlf_profitable(
+; VETO-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; VETO-NEXT: [[ENTRY:.*]]:
+; VETO-NEXT: br label %[[FOR_BODY:.*]]
+; VETO: [[FOR_BODY]]:
+; VETO-NEXT: [[I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; VETO-NEXT: [[BI:%.*]] = add i64 [[I]], -1
+; VETO-NEXT: [[BGEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BI]]
+; VETO-NEXT: [[T:%.*]] = load i32, ptr [[BGEP]], align 4
+; VETO-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i64 0
+; VETO-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; VETO-NEXT: [[TMP2:%.*]] = add <2 x i32> [[TMP1]], <i32 1, i32 4>
+; VETO-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[TMP2]], [[TMP2]]
+; VETO-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP3]], <i32 7, i32 10>
+; VETO-NEXT: [[TMP5:%.*]] = mul <2 x i32> [[TMP4]], [[TMP2]]
+; VETO-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], <i32 11, i32 14>
+; VETO-NEXT: [[TMP7:%.*]] = mul <2 x i32> [[TMP6]], [[TMP4]]
+; VETO-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; VETO-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; VETO-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; VETO-NEXT: [[G1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; VETO-NEXT: [[G3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; VETO-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; VETO-NEXT: store i32 [[TMP8]], ptr [[G0]], align 4
+; VETO-NEXT: [[TMP9:%.*]] = add <2 x i32> [[TMP1]], <i32 2, i32 3>
+; VETO-NEXT: [[TMP10:%.*]] = mul <2 x i32> [[TMP9]], [[TMP9]]
+; VETO-NEXT: [[TMP11:%.*]] = add <2 x i32> [[TMP10]], <i32 8, i32 9>
+; VETO-NEXT: [[TMP12:%.*]] = mul <2 x i32> [[TMP11]], [[TMP9]]
+; VETO-NEXT: [[TMP13:%.*]] = add <2 x i32> [[TMP12]], <i32 12, i32 13>
+; VETO-NEXT: [[TMP14:%.*]] = mul <2 x i32> [[TMP13]], [[TMP11]]
+; VETO-NEXT: store <2 x i32> [[TMP14]], ptr [[G1]], align 4
+; VETO-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; VETO-NEXT: store i32 [[TMP15]], ptr [[G3]], align 4
+; VETO-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; VETO-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; VETO-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; VETO: [[FOR_END]]:
+; VETO-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 1, %entry ], [ %i.next, %for.body ]
+ %bi = add i64 %i, -1
+ %bgep = getelementptr inbounds i32, ptr %A, i64 %bi
+ %t = load i32, ptr %bgep, align 4
+
+ %a0 = add i32 %t, 1
+ %b0 = mul i32 %a0, %a0
+ %c0 = add i32 %b0, 7
+ %d0 = mul i32 %c0, %a0
+ %e0 = add i32 %d0, 11
+ %f0 = mul i32 %e0, %c0
+
+ %a1 = add i32 %t, 2
+ %b1 = mul i32 %a1, %a1
+ %c1 = add i32 %b1, 8
+ %d1 = mul i32 %c1, %a1
+ %e1 = add i32 %d1, 12
+ %f1 = mul i32 %e1, %c1
+
+ %a2 = add i32 %t, 3
+ %b2 = mul i32 %a2, %a2
+ %c2 = add i32 %b2, 9
+ %d2 = mul i32 %c2, %a2
+ %e2 = add i32 %d2, 13
+ %f2 = mul i32 %e2, %c2
+
+ %a3 = add i32 %t, 4
+ %b3 = mul i32 %a3, %a3
+ %c3 = add i32 %b3, 10
+ %d3 = mul i32 %c3, %a3
+ %e3 = add i32 %d3, 14
+ %f3 = mul i32 %e3, %c3
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+ %g0 = getelementptr inbounds i32, ptr %A, i64 %i
+ %g1 = getelementptr inbounds i32, ptr %A, i64 %i1
+ %g2 = getelementptr inbounds i32, ptr %A, i64 %i2
+ %g3 = getelementptr inbounds i32, ptr %A, i64 %i3
+ store i32 %f0, ptr %g0, align 4
+ store i32 %f1, ptr %g1, align 4
+ store i32 %f2, ptr %g2, align 4
+ store i32 %f3, ptr %g3, align 4
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 226f8068626da..d55aad715fd08 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1544,6 +1544,7 @@ void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
emitProcessorProp(OS, PM.ModelDef, "LoadLatency", ',');
emitProcessorProp(OS, PM.ModelDef, "HighLatency", ',');
emitProcessorProp(OS, PM.ModelDef, "MispredictPenalty", ',');
+ emitProcessorProp(OS, PM.ModelDef, "StoreLoadForwardingPenalty", ',');
bool PostRAScheduler =
(PM.ModelDef ? PM.ModelDef->getValueAsBit("PostRAScheduler") : false);
>From 6766960ddae4347a14f7aea98f43f243734ef3d1 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 6 Aug 2026 15:05:19 +0530
Subject: [PATCH 2/4] [SLP] Address review comments on STLF cost modeling -
Enumerate candidate loads directly from the tree's load/gather nodes
instead of a whole-tree operand walk. - Move the scalable-type bailout early
and drop the redundant VF cast. - Extend the no-conflict cache seeding to all
divisors of VF (covers non-power-of-2 VFs), and add a
-slp-vectorize-non-power-of-2 test. - Define the fallback STLF penalty as an
InstructionCost in the TTI layer and drop the MCSchedule.h include from
TargetTransformInfoImpl.h.
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 10 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 58 ++++----
.../X86/store-load-forward-conflict.ll | 92 +++++--------
.../X86/store-load-forward-non-power-of-2.ll | 125 ++++++++++++++++++
4 files changed, 187 insertions(+), 98 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-non-power-of-2.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 2d402429a8b8a..f27c20776c24f 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -23,7 +23,6 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
-#include "llvm/MC/MCSchedule.h"
#include <optional>
#include <utility>
@@ -684,10 +683,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual InstructionCost getStoreLoadForwardingConflictCost(
Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
- // No subtarget scheduling model is available here, so fall back to the
- // conservative default STLF stall penalty (targets with a real model go
- // through BasicTTIImpl and use their own value instead).
- return InstructionCost(MCSchedModel::DefaultStoreLoadForwardingPenalty);
+ // No subtarget scheduling model is available here, so fall back to a
+ // conservative default STLF stall penalty (~10 cycles). Targets with a real
+ // scheduling model go through BasicTTIImpl and use their own value instead.
+ constexpr unsigned DefaultStoreLoadForwardingConflictCost = 10;
+ return InstructionCost(DefaultStoreLoadForwardingConflictCost);
}
virtual std::optional<unsigned>
getCacheSize(TargetTransformInfo::CacheLevel Level) const {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fe5c1b7338e58..2f1dc8d58c67a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28300,18 +28300,20 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
auto CacheAndReturn = [&](bool Result) -> bool {
StlfConflictCache[Key] = Result;
- // No conflict at VF implies none at any smaller power-of-2 VF, so seed
- // those entries too. Conflicts do not propagate downward.
- if (!Result && isPowerOf2_32(VF))
- for (unsigned V = VF / 2; V >= 2; V /= 2)
- StlfConflictCache.try_emplace(std::make_pair(FirstStore, V), false);
+ // No conflict at VF implies none at any smaller VF that divides VF, so seed
+ // those entries too. The recency window is VF-independent (so it stays
+ // satisfied for any smaller width), and the distance remains a multiple of
+ // V * ElementSize only for divisors of VF. This covers the power-of-2
+ // ladder (each smaller power of 2 divides the larger) and any
+ // non-power-of-2 VF the caller may probe. Conflicts do not propagate
+ // downward.
+ if (!Result)
+ for (unsigned V = 2; V < VF; ++V)
+ if (VF % V == 0)
+ StlfConflictCache.try_emplace(std::make_pair(FirstStore, V), false);
return Result;
};
- Loop *L = LI->getLoopFor(FirstStore->getParent());
- if (!L)
- return CacheAndReturn(false);
-
Type *ValueTy = FirstStore->getValueOperand()->getType();
TypeSize StoreSize = DL->getTypeStoreSize(ValueTy);
if (StoreSize.isScalable())
@@ -28319,39 +28321,33 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
uint64_t ElementSize = StoreSize.getFixedValue();
if (ElementSize == 0)
return CacheAndReturn(false);
- uint64_t VectorStoreBytes = uint64_t(VF) * ElementSize;
+
+ // Store-to-load forwarding hazards are a loop-carried concern.
+ if (!LI->getLoopFor(FirstStore->getParent()))
+ return CacheAndReturn(false);
+
+ uint64_t VectorStoreBytes = VF * ElementSize;
LLVM_DEBUG(dbgs() << "SLP: STLF check: VF=" << VF
<< " ElementSize=" << ElementSize
<< " VectorStoreBytes=" << VectorStoreBytes << "\n");
- // Enumerate candidate loads from the tree, not the whole loop. A conflicting
- // load feeds the vectorized nodes, so it is reachable by walking operands
- // from the tree scalars (including through gather leaves such as splats).
- // Stay inside the loop to bound the walk by the tree's cone.
+ // Enumerate candidate loads directly from the tree's load and gather nodes: a
+ // conflicting load is either widened (a load node) or packed into a gather
+ // leaf (e.g. a splat), so scanning those node kinds is sufficient.
Value *StoreBase = getUnderlyingObject(FirstStore->getPointerOperand());
SmallPtrSet<LoadInst *, 8> CandidateLoads;
- SmallPtrSet<const Value *, 32> Visited;
- SmallVector<Value *, 32> Worklist;
for (const std::unique_ptr<TreeEntry> &TEPtr : VectorizableTree) {
const TreeEntry *TE = TEPtr.get();
if (DeletedNodes.contains(TE))
continue;
- Worklist.append(TE->Scalars.begin(), TE->Scalars.end());
- }
- while (!Worklist.empty()) {
- Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- if (auto *LoadI = dyn_cast<LoadInst>(V)) {
- if (LoadI->isSimple() &&
- getUnderlyingObject(LoadI->getPointerOperand()) == StoreBase)
- CandidateLoads.insert(LoadI);
- continue;
- }
- auto *I = dyn_cast<Instruction>(V);
- if (!I || !L->contains(I))
+ if (!TE->isGather() &&
+ !(TE->hasState() && TE->getOpcode() == Instruction::Load))
continue;
- Worklist.append(I->op_begin(), I->op_end());
+ for (Value *V : TE->Scalars)
+ if (auto *LoadI = dyn_cast<LoadInst>(V))
+ if (LoadI->isSimple() &&
+ getUnderlyingObject(LoadI->getPointerOperand()) == StoreBase)
+ CandidateLoads.insert(LoadI);
}
if (CandidateLoads.empty())
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index 906c560635fc0..f17d390ee6220 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -415,71 +415,39 @@ for.end:
}
;
-; Test 7: Multiple conflicting loads in the same loop.
+; Test 7: Loads that feed scalar arithmetic below a gather/splat leaf.
;
-; Both A[i-5] (distance 20 bytes) and A[i-7] (distance 28 bytes) are
-; misaligned to a 16-byte vector store and within the safety window. The
-; check fires on the first one encountered and the store entry is penalized,
-; keeping the chain scalar.
+; A[i-5] and A[i-7] would be misaligned to a 16-byte vector store, but they
+; only feed the scalar %sum, which is then broadcast. Candidate loads are
+; enumerated from the tree's load and gather nodes, so loads sitting below a
+; gather leaf (here, under %sum) are not seen and no penalty is charged; the
+; chain vectorizes. This documents the reachability limit of the node-based
+; enumeration.
;
define void @stlf_multiple_conflicting_loads(ptr noalias %A, i64 %n) {
-; STLF-ON-LABEL: define void @stlf_multiple_conflicting_loads(
-; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; STLF-ON-NEXT: [[ENTRY:.*]]:
-; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
-; STLF-ON: [[FOR_BODY]]:
-; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; STLF-ON-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
-; STLF-ON-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
-; STLF-ON-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
-; STLF-ON-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
-; STLF-ON-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
-; STLF-ON-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
-; STLF-ON-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
-; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[SUM]], 1
-; STLF-ON-NEXT: [[T2:%.*]] = add nsw i32 [[SUM]], 2
-; STLF-ON-NEXT: [[T3:%.*]] = add nsw i32 [[SUM]], 3
-; STLF-ON-NEXT: [[T4:%.*]] = add nsw i32 [[SUM]], 4
-; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
-; STLF-ON-NEXT: [[I2:%.*]] = add nuw nsw i64 [[I]], 2
-; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
-; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
-; STLF-ON-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I2]]
-; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
-; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
-; STLF-ON-NEXT: store i32 [[T2]], ptr [[GEP1]], align 4
-; STLF-ON-NEXT: store i32 [[T3]], ptr [[GEP2]], align 4
-; STLF-ON-NEXT: store i32 [[T4]], ptr [[GEP3]], align 4
-; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; STLF-ON: [[FOR_END]]:
-; STLF-ON-NEXT: ret void
-;
-; STLF-OFF-LABEL: define void @stlf_multiple_conflicting_loads(
-; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; STLF-OFF-NEXT: [[ENTRY:.*]]:
-; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
-; STLF-OFF: [[FOR_BODY]]:
-; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
-; STLF-OFF-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
-; STLF-OFF-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
-; STLF-OFF-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
-; STLF-OFF-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
-; STLF-OFF-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
-; STLF-OFF-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
-; STLF-OFF-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
-; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
-; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i64 0
-; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
-; STLF-OFF-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
-; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
-; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
-; STLF-OFF: [[FOR_END]]:
-; STLF-OFF-NEXT: ret void
+; CHECK-LABEL: define void @stlf_multiple_conflicting_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[BACK5_IDX:%.*]] = sub i64 [[I]], 5
+; CHECK-NEXT: [[BACK5_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK5_IDX]]
+; CHECK-NEXT: [[T5:%.*]] = load i32, ptr [[BACK5_GEP]], align 4
+; CHECK-NEXT: [[BACK7_IDX:%.*]] = sub i64 [[I]], 7
+; CHECK-NEXT: [[BACK7_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK7_IDX]]
+; CHECK-NEXT: [[T7:%.*]] = load i32, ptr [[BACK7_GEP]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add nsw i32 [[T5]], [[T7]]
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[SUM]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-non-power-of-2.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-non-power-of-2.ll
new file mode 100644
index 0000000000000..74433ed96fdd8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-non-power-of-2.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -S \
+; RUN: -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-ON
+; RUN: opt < %s -passes=slp-vectorizer -slp-vectorize-non-power-of-2 \
+; RUN: -slp-store-load-forward-check=false -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-OFF
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;
+; Store-to-load forwarding hazard at a non-power-of-2 vector factor.
+;
+; for (int i = 7; i < n; i += 7) {
+; int t = A[i-5]; ; load 20 bytes (5*4) before the store base
+; A[i] = t + 1;
+; ...
+; A[i+6] = t + 7;
+; }
+;
+; With -slp-vectorize-non-power-of-2, SLP first probes VF=7. The backward load
+; A[i-5] is 20 bytes behind the store base:
+; 20 % (7*4=28) = 20 -> misaligned, load straddles two pending vector stores
+; 20 / 28 = 0 -> store still hot in the store buffer
+; so the STLF penalty prices the VF=7 (and the VF=4 base) widening out. With the
+; check disabled the whole chain widens to a single <7 x i32> store. This
+; exercises the hazard check at a non-power-of-2 VF and the divisor-based cache
+; propagation.
+;
+define void @stlf_non_power_of_2(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @stlf_non_power_of_2(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-ON-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-ON-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-ON-NEXT: [[T1:%.*]] = add nsw i32 [[T]], 1
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i32 [[T1]], ptr [[GEP0]], align 4
+; STLF-ON-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[T]], i64 0
+; STLF-ON-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 2, i32 3>
+; STLF-ON-NEXT: store <2 x i32> [[TMP2]], ptr [[GEP1]], align 4
+; STLF-ON-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[T]], i64 0
+; STLF-ON-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; STLF-ON-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[TMP4]], <i32 4, i32 5, i32 6, i32 7>
+; STLF-ON-NEXT: store <4 x i32> [[TMP5]], ptr [[GEP3]], align 4
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 7
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_non_power_of_2(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 7, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BACK_IDX:%.*]] = sub i64 [[I]], 5
+; STLF-OFF-NEXT: [[BACK_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[BACK_IDX]]
+; STLF-OFF-NEXT: [[T:%.*]] = load i32, ptr [[BACK_GEP]], align 4
+; STLF-OFF-NEXT: [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <7 x i32> poison, i32 [[T]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <7 x i32> [[TMP0]], <7 x i32> poison, <7 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP2:%.*]] = add nsw <7 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; STLF-OFF-NEXT: store <7 x i32> [[TMP2]], ptr [[GEP0]], align 4
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 7
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 7, %entry ], [ %i.next, %for.body ]
+ %back.idx = sub i64 %i, 5
+ %back.gep = getelementptr inbounds i32, ptr %A, i64 %back.idx
+ %t = load i32, ptr %back.gep, align 4
+ %t1 = add nsw i32 %t, 1
+ %t2 = add nsw i32 %t, 2
+ %t3 = add nsw i32 %t, 3
+ %t4 = add nsw i32 %t, 4
+ %t5 = add nsw i32 %t, 5
+ %t6 = add nsw i32 %t, 6
+ %t7 = add nsw i32 %t, 7
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+ %i4 = add nuw nsw i64 %i, 4
+ %i5 = add nuw nsw i64 %i, 5
+ %i6 = add nuw nsw i64 %i, 6
+ %gep0 = getelementptr inbounds i32, ptr %A, i64 %i
+ %gep1 = getelementptr inbounds i32, ptr %A, i64 %i1
+ %gep2 = getelementptr inbounds i32, ptr %A, i64 %i2
+ %gep3 = getelementptr inbounds i32, ptr %A, i64 %i3
+ %gep4 = getelementptr inbounds i32, ptr %A, i64 %i4
+ %gep5 = getelementptr inbounds i32, ptr %A, i64 %i5
+ %gep6 = getelementptr inbounds i32, ptr %A, i64 %i6
+ store i32 %t1, ptr %gep0, align 4
+ store i32 %t2, ptr %gep1, align 4
+ store i32 %t3, ptr %gep2, align 4
+ store i32 %t4, ptr %gep3, align 4
+ store i32 %t5, ptr %gep4, align 4
+ store i32 %t6, ptr %gep5, align 4
+ store i32 %t7, ptr %gep6, align 4
+ %i.next = add nuw nsw i64 %i, 7
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From e8300d5744a85326e8a64e1ceecf2c7bd566da4e Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Wed, 12 Aug 2026 13:55:36 +0530
Subject: [PATCH 3/4] [SLP] Detect wider-than-store loads in STLF check; dedup
penalty default Extend MemoryDepChecker::isStoreLoadForwardingConflict with
an optional LoadElementSize: an aligned load wider than the widened store
still straddles two wide stores, which the misalignment-only test missed.
LAA's call is unchanged (default 0), keeping it NFC; SLP passes the load
width. Also dedup the store-load-forwarding penalty default via MCSchedModel
and drop an unused DenseMap.h include.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 38 +++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 11 +-
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 1 -
.../Transforms/Vectorize/SLPVectorizer.cpp | 11 +-
.../X86/store-load-forward-conflict.ll | 211 ++++++++++++++++++
5 files changed, 251 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index dbd0248f2e58f..d021351f12164 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -236,22 +236,36 @@ class MemoryDepChecker {
}
/// Returns true if a memory dependence at byte distance \p Distance between
- /// a store and load (both with element size \p TypeByteSize bytes) would
- /// prevent store-to-load forwarding when the store is widened to
- /// \p VectorStoreSize bytes.
+ /// a store and load would prevent store-to-load forwarding when the store is
+ /// widened to \p VectorStoreSize bytes. \p TypeByteSize is the store element
+ /// size; \p LoadElementSize, when non-zero, is the load's access width.
///
- /// The predicate fires when (a) the load is misaligned w.r.t. the widened
- /// store window (\c Distance is not a multiple of \p VectorStoreSize), and
- /// (b) the conflicting store is still likely to be in the store buffer
- /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations).
- /// Both couldPreventStoreLoadForward and SLPVectorizer use this as their
- /// core STLF cost-model gate.
+ /// The conflicting store must still be likely to be in the store buffer
+ /// (\c Distance / VectorStoreSize is below 8 * TypeByteSize iterations), and
+ /// the load must overlap two widened stores, which happens when either:
+ /// (a) the load is misaligned w.r.t. the widened store window
+ /// (\c Distance is not a multiple of \p VectorStoreSize), or
+ /// (b) the load starts aligned but is itself wider than the widened store
+ /// window (\p LoadElementSize > \p VectorStoreSize), so it overruns
+ /// into the next widened store.
+ /// Both couldPreventStoreLoadForward and SLPVectorizer use this as their core
+ /// STLF cost-model gate. LAA omits \p LoadElementSize (the load and store
+ /// share the element size there, so (b) can never add anything).
static bool isStoreLoadForwardingConflict(uint64_t Distance,
uint64_t VectorStoreSize,
- uint64_t TypeByteSize) {
+ uint64_t TypeByteSize,
+ uint64_t LoadElementSize = 0) {
+ if (VectorStoreSize == 0)
+ return false;
const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
- return VectorStoreSize != 0 && Distance % VectorStoreSize != 0 &&
- Distance / VectorStoreSize < NumItersForStoreLoadThroughMemory;
+ if (Distance / VectorStoreSize >= NumItersForStoreLoadThroughMemory)
+ return false;
+ // (a) Misaligned load: it straddles two widened stores.
+ if (Distance % VectorStoreSize != 0)
+ return true;
+ // (b) Aligned load that is wider than the widened store still overruns
+ // into the next widened store (LoadElementSize == 0 disables this term).
+ return LoadElementSize > VectorStoreSize;
}
/// Return safe power-of-2 number of elements, which do not prevent store-load
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index f27c20776c24f..899445e47c48e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -23,6 +23,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/MC/MCSchedule.h"
#include <optional>
#include <utility>
@@ -683,11 +684,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual InstructionCost getStoreLoadForwardingConflictCost(
Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
- // No subtarget scheduling model is available here, so fall back to a
- // conservative default STLF stall penalty (~10 cycles). Targets with a real
- // scheduling model go through BasicTTIImpl and use their own value instead.
- constexpr unsigned DefaultStoreLoadForwardingConflictCost = 10;
- return InstructionCost(DefaultStoreLoadForwardingConflictCost);
+ // No subtarget scheduling model is available here, so fall back to the same
+ // default store-to-load forwarding penalty the scheduling model uses.
+ // Targets with a real scheduling model go through BasicTTIImpl and use
+ // their own StoreLoadForwardingPenalty instead.
+ return InstructionCost(MCSchedModel::DefaultStoreLoadForwardingPenalty);
}
virtual std::optional<unsigned>
getCacheSize(TargetTransformInfo::CacheLevel Level) const {
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index b35a49c227487..538d62626b37b 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -19,7 +19,6 @@
#define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2f1dc8d58c67a..24f83cc29af5c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28367,14 +28367,19 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
LLVM_DEBUG(dbgs() << "SLP: STLF: load=" << *LoadI << " distance="
<< Distance << " bytes from chain base\n");
- // Conflict if the load is misaligned to the wide store within the recency
- // window.
+ // Conflict if the load overlaps two wide stores within the recency window,
+ // either because it is misaligned or because the load itself is wider than
+ // the wide store and overruns its window.
+ TypeSize LoadTypeSize = DL->getTypeStoreSize(LoadI->getType());
+ uint64_t LoadElementSize =
+ LoadTypeSize.isScalable() ? 0 : LoadTypeSize.getFixedValue();
if (MemoryDepChecker::isStoreLoadForwardingConflict(
- Distance, VectorStoreBytes, ElementSize)) {
+ Distance, VectorStoreBytes, ElementSize, LoadElementSize)) {
LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: "
<< (isVectorized(LoadI) ? "widened" : "scalar")
<< " load, distance " << Distance
<< " bytes, vector store width " << VectorStoreBytes
+ << " bytes, load width " << LoadElementSize
<< " bytes, misalignment "
<< (Distance % VectorStoreBytes) << "\n");
return CacheAndReturn(true);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
index f17d390ee6220..c5aded18fb462 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-conflict.ll
@@ -801,3 +801,214 @@ for.body:
for.end:
ret void
}
+
+;
+; Test 12: Aligned load that is WIDER than the widened store (crossing).
+;
+; i8 store chain, VF=4 => widened store window W = 4 bytes. A backward i64
+; load (element size 8 > 4) sits at aligned Distance = 4: 4 % 4 == 0 (not
+; misaligned), but the load itself overruns its window and straddles two
+; widened stores. The misalignment-only test would miss this; the load-width
+; term catches it, so vectorization must be rejected with the check on.
+;
+define void @stlf_conflict_cross_boundary_wider_load(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @stlf_conflict_cross_boundary_wider_load(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 8, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[B0:%.*]] = sub i64 [[I]], 4
+; STLF-ON-NEXT: [[G0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[B0]]
+; STLF-ON-NEXT: [[X0:%.*]] = load i64, ptr [[G0]], align 1
+; STLF-ON-NEXT: [[B1:%.*]] = add i64 [[I]], 64
+; STLF-ON-NEXT: [[G1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[B1]]
+; STLF-ON-NEXT: [[X1:%.*]] = load i64, ptr [[G1]], align 1
+; STLF-ON-NEXT: [[B2:%.*]] = add i64 [[I]], 128
+; STLF-ON-NEXT: [[G2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[B2]]
+; STLF-ON-NEXT: [[X2:%.*]] = load i64, ptr [[G2]], align 1
+; STLF-ON-NEXT: [[B3:%.*]] = add i64 [[I]], 192
+; STLF-ON-NEXT: [[G3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[B3]]
+; STLF-ON-NEXT: [[X3:%.*]] = load i64, ptr [[G3]], align 1
+; STLF-ON-NEXT: [[T0:%.*]] = trunc i64 [[X0]] to i8
+; STLF-ON-NEXT: [[T1:%.*]] = trunc i64 [[X1]] to i8
+; STLF-ON-NEXT: [[T2:%.*]] = trunc i64 [[X2]] to i8
+; STLF-ON-NEXT: [[T3:%.*]] = trunc i64 [[X3]] to i8
+; STLF-ON-NEXT: [[S0:%.*]] = add i8 [[T0]], 1
+; STLF-ON-NEXT: [[S1:%.*]] = add i8 [[T1]], 2
+; STLF-ON-NEXT: [[S2:%.*]] = add i8 [[T2]], 3
+; STLF-ON-NEXT: [[S3:%.*]] = add i8 [[T3]], 4
+; STLF-ON-NEXT: [[I1:%.*]] = add nuw nsw i64 [[I]], 1
+; STLF-ON-NEXT: [[I2:%.*]] = add nuw nsw i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add nuw nsw i64 [[I]], 3
+; STLF-ON-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I2]]
+; STLF-ON-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i8 [[S0]], ptr [[P0]], align 1
+; STLF-ON-NEXT: store i8 [[S1]], ptr [[P1]], align 1
+; STLF-ON-NEXT: store i8 [[S2]], ptr [[P2]], align 1
+; STLF-ON-NEXT: store i8 [[S3]], ptr [[P3]], align 1
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @stlf_conflict_cross_boundary_wider_load(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[A]], i64 0
+; STLF-OFF-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 8, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[I]], i64 0
+; STLF-OFF-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; STLF-OFF-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP3]], <i64 -4, i64 64, i64 128, i64 192>
+; STLF-OFF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP1]], <4 x i64> [[TMP4]]
+; STLF-OFF-NEXT: [[TMP6:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 1 [[TMP5]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; STLF-OFF-NEXT: [[TMP7:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i8>
+; STLF-OFF-NEXT: [[TMP8:%.*]] = add <4 x i8> [[TMP7]], <i8 1, i8 2, i8 3, i8 4>
+; STLF-OFF-NEXT: store <4 x i8> [[TMP8]], ptr [[P0]], align 1
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 8, %entry ], [ %i.next, %for.body ]
+
+ %b0 = sub i64 %i, 4
+ %g0 = getelementptr inbounds i8, ptr %A, i64 %b0
+ %x0 = load i64, ptr %g0, align 1
+ %b1 = add i64 %i, 64
+ %g1 = getelementptr inbounds i8, ptr %A, i64 %b1
+ %x1 = load i64, ptr %g1, align 1
+ %b2 = add i64 %i, 128
+ %g2 = getelementptr inbounds i8, ptr %A, i64 %b2
+ %x2 = load i64, ptr %g2, align 1
+ %b3 = add i64 %i, 192
+ %g3 = getelementptr inbounds i8, ptr %A, i64 %b3
+ %x3 = load i64, ptr %g3, align 1
+
+ %t0 = trunc i64 %x0 to i8
+ %t1 = trunc i64 %x1 to i8
+ %t2 = trunc i64 %x2 to i8
+ %t3 = trunc i64 %x3 to i8
+
+ %s0 = add i8 %t0, 1
+ %s1 = add i8 %t1, 2
+ %s2 = add i8 %t2, 3
+ %s3 = add i8 %t3, 4
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+
+ %p0 = getelementptr inbounds i8, ptr %A, i64 %i
+ %p1 = getelementptr inbounds i8, ptr %A, i64 %i1
+ %p2 = getelementptr inbounds i8, ptr %A, i64 %i2
+ %p3 = getelementptr inbounds i8, ptr %A, i64 %i3
+
+ store i8 %s0, ptr %p0, align 1
+ store i8 %s1, ptr %p1, align 1
+ store i8 %s2, ptr %p2, align 1
+ store i8 %s3, ptr %p3, align 1
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+;
+; Test 13: Aligned load wider than the element but CONTAINED (no crossing).
+;
+; Same shape as Test 12, but the backward load is i32 (element size 4 == W).
+; Starting aligned, it exactly fills one widened store window and does not
+; overrun into the next one, so it is NOT a forwarding conflict. Vectorization
+; must happen regardless of the STLF check, proving the load-width term does
+; not over-reject aligned, contained loads.
+;
+define void @stlf_no_conflict_wider_contained_load(ptr noalias %A, i64 %n) {
+; CHECK-LABEL: define void @stlf_no_conflict_wider_contained_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[A]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 8, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[I]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP3]], <i64 -4, i64 64, i64 128, i64 192>
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP1]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 [[TMP5]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i8> [[TMP7]], <i8 1, i8 2, i8 3, i8 4>
+; CHECK-NEXT: store <4 x i8> [[TMP8]], ptr [[P0]], align 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 8, %entry ], [ %i.next, %for.body ]
+
+ %b0 = sub i64 %i, 4
+ %g0 = getelementptr inbounds i8, ptr %A, i64 %b0
+ %x0 = load i32, ptr %g0, align 1
+ %b1 = add i64 %i, 64
+ %g1 = getelementptr inbounds i8, ptr %A, i64 %b1
+ %x1 = load i32, ptr %g1, align 1
+ %b2 = add i64 %i, 128
+ %g2 = getelementptr inbounds i8, ptr %A, i64 %b2
+ %x2 = load i32, ptr %g2, align 1
+ %b3 = add i64 %i, 192
+ %g3 = getelementptr inbounds i8, ptr %A, i64 %b3
+ %x3 = load i32, ptr %g3, align 1
+
+ %t0 = trunc i32 %x0 to i8
+ %t1 = trunc i32 %x1 to i8
+ %t2 = trunc i32 %x2 to i8
+ %t3 = trunc i32 %x3 to i8
+
+ %s0 = add i8 %t0, 1
+ %s1 = add i8 %t1, 2
+ %s2 = add i8 %t2, 3
+ %s3 = add i8 %t3, 4
+
+ %i1 = add nuw nsw i64 %i, 1
+ %i2 = add nuw nsw i64 %i, 2
+ %i3 = add nuw nsw i64 %i, 3
+
+ %p0 = getelementptr inbounds i8, ptr %A, i64 %i
+ %p1 = getelementptr inbounds i8, ptr %A, i64 %i1
+ %p2 = getelementptr inbounds i8, ptr %A, i64 %i2
+ %p3 = getelementptr inbounds i8, ptr %A, i64 %i3
+
+ store i8 %s0, ptr %p0, align 1
+ store i8 %s1, ptr %p1, align 1
+ store i8 %s2, ptr %p2, align 1
+ store i8 %s3, ptr %p3, align 1
+
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
>From 8e06b065d3520e571664db3bc6678d912df0f8ca Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 13 Aug 2026 10:41:48 +0530
Subject: [PATCH 4/4] [SLP] Address review: widened load width, same-loop
guard, skip gathered nodes
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 12 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 10 ++
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 +++-
.../X86/store-load-forward-widened-load.ll | 132 ++++++++++++++++++
4 files changed, 166 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-widened-load.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 899445e47c48e..cc8cfd2785aea 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -23,7 +23,6 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
-#include "llvm/MC/MCSchedule.h"
#include <optional>
#include <utility>
@@ -682,14 +681,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
virtual unsigned getCacheLineSize() const { return 0; }
+ // Defined out of line in TargetTransformInfo.cpp so this widely-included
+ // header does not need to pull in llvm/MC/MCSchedule.h just to reference the
+ // default store-to-load forwarding penalty.
virtual InstructionCost getStoreLoadForwardingConflictCost(
- Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
- // No subtarget scheduling model is available here, so fall back to the same
- // default store-to-load forwarding penalty the scheduling model uses.
- // Targets with a real scheduling model go through BasicTTIImpl and use
- // their own StoreLoadForwardingPenalty instead.
- return InstructionCost(MCSchedModel::DefaultStoreLoadForwardingPenalty);
- }
+ Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const;
virtual std::optional<unsigned>
getCacheSize(TargetTransformInfo::CacheLevel Level) const {
switch (Level) {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index b4a57d1c00635..db1c6136551d1 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -20,6 +20,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/InitializePasses.h"
+#include "llvm/MC/MCSchedule.h"
#include "llvm/Support/CommandLine.h"
#include <optional>
#include <utility>
@@ -1600,6 +1601,15 @@ bool TargetTransformInfo::isUniform(const Instruction *I,
TargetTransformInfoImplBase::~TargetTransformInfoImplBase() = default;
+InstructionCost TargetTransformInfoImplBase::getStoreLoadForwardingConflictCost(
+ Type *VecTy, TargetTransformInfo::TargetCostKind CostKind) const {
+ // No subtarget scheduling model is available here, so fall back to the same
+ // default store-to-load forwarding penalty the scheduling model uses. Targets
+ // with a real scheduling model go through BasicTTIImpl and use their own
+ // StoreLoadForwardingPenalty instead.
+ return InstructionCost(MCSchedModel::DefaultStoreLoadForwardingPenalty);
+}
+
TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
TargetIRAnalysis::TargetIRAnalysis(
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 24f83cc29af5c..e5b0de4586e4a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28323,7 +28323,8 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
return CacheAndReturn(false);
// Store-to-load forwarding hazards are a loop-carried concern.
- if (!LI->getLoopFor(FirstStore->getParent()))
+ const Loop *StoreL = LI->getLoopFor(FirstStore->getParent());
+ if (!StoreL)
return CacheAndReturn(false);
uint64_t VectorStoreBytes = VF * ElementSize;
@@ -28338,7 +28339,7 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
SmallPtrSet<LoadInst *, 8> CandidateLoads;
for (const std::unique_ptr<TreeEntry> &TEPtr : VectorizableTree) {
const TreeEntry *TE = TEPtr.get();
- if (DeletedNodes.contains(TE))
+ if (DeletedNodes.contains(TE) || TransformedToGatherNodes.contains(TE))
continue;
if (!TE->isGather() &&
!(TE->hasState() && TE->getOpcode() == Instruction::Load))
@@ -28356,6 +28357,9 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
// For each candidate load, the widened chain becomes one wide store at the
// base; check whether the load straddles two such wide stores.
for (LoadInst *LoadI : CandidateLoads) {
+ // Only loads in the store's loop share its loop-carried dependence.
+ if (LI->getLoopFor(LoadI->getParent()) != StoreL)
+ continue;
std::optional<int64_t> Diff =
getPointersDiff(ValueTy, FirstStore->getPointerOperand(),
LoadI->getType(), LoadI->getPointerOperand(), *DL, *SE,
@@ -28367,12 +28371,23 @@ bool BoUpSLP::findStoreLoadForwardingConflict(StoreInst *BaseStore,
LLVM_DEBUG(dbgs() << "SLP: STLF: load=" << *LoadI << " distance="
<< Distance << " bytes from chain base\n");
- // Conflict if the load overlaps two wide stores within the recency window,
- // either because it is misaligned or because the load itself is wider than
- // the wide store and overruns its window.
+ // A widened (regularly vectorized) load accesses the whole vector at once,
+ // so its effective width is VectorFactor * element size, not one element.
+ // Such a wide load can straddle two wide stores even when perfectly
+ // aligned, which the misalignment-only test would miss.
TypeSize LoadTypeSize = DL->getTypeStoreSize(LoadI->getType());
uint64_t LoadElementSize =
LoadTypeSize.isScalable() ? 0 : LoadTypeSize.getFixedValue();
+ for (const TreeEntry *LTE : getTreeEntries(LoadI)) {
+ if (LTE->State == TreeEntry::Vectorize &&
+ LTE->getOpcode() == Instruction::Load) {
+ LoadElementSize *= LTE->getVectorFactor();
+ break;
+ }
+ }
+ // Conflict if the load overlaps two wide stores within the recency window,
+ // either because it is misaligned or because the load itself is wider than
+ // the wide store and overruns its window.
if (MemoryDepChecker::isStoreLoadForwardingConflict(
Distance, VectorStoreBytes, ElementSize, LoadElementSize)) {
LLVM_DEBUG(dbgs() << "SLP: Store-load forwarding conflict: "
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-widened-load.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-widened-load.ll
new file mode 100644
index 0000000000000..890b06a477aff
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-load-forward-widened-load.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-ON
+; RUN: opt < %s -passes=slp-vectorizer -slp-store-load-forward-check=false -S \
+; RUN: -mtriple=x86_64-- -mcpu=znver5 \
+; RUN: | FileCheck %s --check-prefixes=CHECK,STLF-OFF
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; A widened load can straddle two widened stores even when it is perfectly
+; aligned to the store window, because the load itself is wider than that
+; window. Here four contiguous i32 loads become a 16-byte <4 x i32> load, and
+; the truncated results feed four i8 stores that become a 4-byte <4 x i8>
+; store. The load base is 16 bytes behind the store base:
+; 16 % 4 = 0 -> aligned to the widened store window (misalignment term (a)
+; does not fire), but
+; load width 16 > 4 = widened store width -> the aligned-but-wider term (b)
+; fires: the wide load overruns into the next widened store.
+; 16 / 4 = 4 iterations -> the store is still hot in the store buffer.
+; So SLP must price in the store-to-load forwarding penalty and leave the chain
+; scalar with the check on; with the check off it widens both load and store.
+; This case is reachable only because the STLF gate uses the *widened* load
+; width (VectorFactor * element size) rather than a single element.
+define void @widened_load_crosses(ptr noalias %A, i64 %n) {
+; STLF-ON-LABEL: define void @widened_load_crosses(
+; STLF-ON-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-ON-NEXT: [[ENTRY:.*]]:
+; STLF-ON-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-ON: [[FOR_BODY]]:
+; STLF-ON-NEXT: [[I:%.*]] = phi i64 [ 64, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-ON-NEXT: [[BM16:%.*]] = add i64 [[I]], -16
+; STLF-ON-NEXT: [[BM12:%.*]] = add i64 [[I]], -12
+; STLF-ON-NEXT: [[BM8:%.*]] = add i64 [[I]], -8
+; STLF-ON-NEXT: [[BM4:%.*]] = add i64 [[I]], -4
+; STLF-ON-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[BM16]]
+; STLF-ON-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[BM12]]
+; STLF-ON-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[BM8]]
+; STLF-ON-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[BM4]]
+; STLF-ON-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; STLF-ON-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; STLF-ON-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; STLF-ON-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; STLF-ON-NEXT: [[A0:%.*]] = add i32 [[L0]], 1
+; STLF-ON-NEXT: [[A1:%.*]] = add i32 [[L1]], 1
+; STLF-ON-NEXT: [[A2:%.*]] = add i32 [[L2]], 1
+; STLF-ON-NEXT: [[A3:%.*]] = add i32 [[L3]], 1
+; STLF-ON-NEXT: [[T0:%.*]] = trunc i32 [[A0]] to i8
+; STLF-ON-NEXT: [[T1:%.*]] = trunc i32 [[A1]] to i8
+; STLF-ON-NEXT: [[T2:%.*]] = trunc i32 [[A2]] to i8
+; STLF-ON-NEXT: [[T3:%.*]] = trunc i32 [[A3]] to i8
+; STLF-ON-NEXT: [[I1:%.*]] = add i64 [[I]], 1
+; STLF-ON-NEXT: [[I2:%.*]] = add i64 [[I]], 2
+; STLF-ON-NEXT: [[I3:%.*]] = add i64 [[I]], 3
+; STLF-ON-NEXT: [[S0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; STLF-ON-NEXT: [[S1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I1]]
+; STLF-ON-NEXT: [[S2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I2]]
+; STLF-ON-NEXT: [[S3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I3]]
+; STLF-ON-NEXT: store i8 [[T0]], ptr [[S0]], align 1
+; STLF-ON-NEXT: store i8 [[T1]], ptr [[S1]], align 1
+; STLF-ON-NEXT: store i8 [[T2]], ptr [[S2]], align 1
+; STLF-ON-NEXT: store i8 [[T3]], ptr [[S3]], align 1
+; STLF-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-ON-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-ON-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-ON: [[FOR_END]]:
+; STLF-ON-NEXT: ret void
+;
+; STLF-OFF-LABEL: define void @widened_load_crosses(
+; STLF-OFF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; STLF-OFF-NEXT: [[ENTRY:.*]]:
+; STLF-OFF-NEXT: br label %[[FOR_BODY:.*]]
+; STLF-OFF: [[FOR_BODY]]:
+; STLF-OFF-NEXT: [[I:%.*]] = phi i64 [ 64, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; STLF-OFF-NEXT: [[BM16:%.*]] = add i64 [[I]], -16
+; STLF-OFF-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[BM16]]
+; STLF-OFF-NEXT: [[S0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; STLF-OFF-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[P0]], align 4
+; STLF-OFF-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], splat (i32 1)
+; STLF-OFF-NEXT: [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i8>
+; STLF-OFF-NEXT: store <4 x i8> [[TMP2]], ptr [[S0]], align 1
+; STLF-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 4
+; STLF-OFF-NEXT: [[CMP:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; STLF-OFF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; STLF-OFF: [[FOR_END]]:
+; STLF-OFF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 64, %entry ], [ %i.next, %for.body ]
+ %bm16 = add i64 %i, -16
+ %bm12 = add i64 %i, -12
+ %bm8 = add i64 %i, -8
+ %bm4 = add i64 %i, -4
+ %p0 = getelementptr inbounds i8, ptr %A, i64 %bm16
+ %p1 = getelementptr inbounds i8, ptr %A, i64 %bm12
+ %p2 = getelementptr inbounds i8, ptr %A, i64 %bm8
+ %p3 = getelementptr inbounds i8, ptr %A, i64 %bm4
+ %l0 = load i32, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %a0 = add i32 %l0, 1
+ %a1 = add i32 %l1, 1
+ %a2 = add i32 %l2, 1
+ %a3 = add i32 %l3, 1
+ %t0 = trunc i32 %a0 to i8
+ %t1 = trunc i32 %a1 to i8
+ %t2 = trunc i32 %a2 to i8
+ %t3 = trunc i32 %a3 to i8
+ %i1 = add i64 %i, 1
+ %i2 = add i64 %i, 2
+ %i3 = add i64 %i, 3
+ %s0 = getelementptr inbounds i8, ptr %A, i64 %i
+ %s1 = getelementptr inbounds i8, ptr %A, i64 %i1
+ %s2 = getelementptr inbounds i8, ptr %A, i64 %i2
+ %s3 = getelementptr inbounds i8, ptr %A, i64 %i3
+ store i8 %t0, ptr %s0, align 1
+ store i8 %t1, ptr %s1, align 1
+ store i8 %t2, ptr %s2, align 1
+ store i8 %t3, ptr %s3, align 1
+ %i.next = add nuw nsw i64 %i, 4
+ %cmp = icmp slt i64 %i.next, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-commits
mailing list