[llvm] [LV] Factor costInterleaveGatherScatter (NFC) (PR #215857)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 00:27:08 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/215857
>From 5a20a69636655f24fa452635a32e3e846fd616a5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 12 Aug 2026 18:14:21 +0100
Subject: [PATCH 1/3] [LV] Factor costInterleaveGatherScatter (NFC)
The motivation for factoring out a costInterleaveGatherScatter that
compares the cost of interleaving versus that of a gather-scatter is for
re-use in a follow-up doing VPlan-based gather-scatter-widening.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 16 +--
.../Vectorize/LoopVectorizationPlanner.h | 7 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 127 +++++++++---------
3 files changed, 70 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index ff9b9171d8c8c..64136846a80bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -146,19 +146,11 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
: TTI.isLegalMaskedStore(ScalarTy, Alignment, AddressSpace));
}
-bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
- ElementCount VF) const {
- bool LI = isa<LoadInst>(V);
- bool SI = isa<StoreInst>(V);
- if (!LI && !SI)
- return false;
- auto *Ty = getLoadStoreType(V);
- Align Align = getLoadStoreAlignment(V);
- if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
+bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const {
return ForceTargetSupportsGatherScatterOps ||
- (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
- (SI && TTI.isLegalMaskedScatter(Ty, Align));
+ (IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
+ : TTI.isLegalMaskedScatter(ScalarTy, Alignment));
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..68eebc8a1c3ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -787,9 +787,10 @@ class VFSelectionContext {
bool isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy, Align Alignment,
unsigned AddressSpace) const;
- /// Returns true if the target machine can represent \p V as a masked gather
- /// or scatter operation.
- bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
+ /// Returns true if the target machine supports a gather (if \p IsLoad)
+ /// or scatter of scalar type \p ScalarTy with \p Alignment.
+ bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const;
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d892ff2a0fa2b..c7b9a815e420e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1034,11 +1034,11 @@ class LoopVectorizationCostModel {
/// every iteration of the loop header.
inline uint64_t
getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
- const BasicBlock *BB);
+ const BasicBlock *BB) const;
/// Returns true if an artificially high cost for emulated masked memrefs
/// should be used.
- bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
+ bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
/// Return the costs for our two available strategies for lowering a
/// div/rem operation which requires speculating at least one lane.
@@ -1063,6 +1063,10 @@ class LoopVectorizationCostModel {
/// consecutive or part of an interleave group.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
+ /// Returns true if the target machine supports gather or scatter for \p I's
+ /// data type and alignment.
+ bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
+
/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) const {
return InterleaveInfo.isInterleaved(Instr);
@@ -1348,17 +1352,47 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ /// Pick between interleave and gather-scatter based on cost. Returns a pair
+ /// of widening decision along with corresponding cost.
+ std::pair<InstWidening, InstructionCost>
+ costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ InstructionCost InterleaveCost = InstructionCost::getInvalid();
+ unsigned NumAccesses = 1;
+ if (isAccessInterleaved(I)) {
+ const auto *Group = getInterleavedAccessGroup(I);
+ assert(Group && "Fail to get an interleaved access group.");
+
+ if (interleavedAccessCanBeWidened(I, VF)) {
+ NumAccesses = Group->getNumMembers();
+ InterleaveCost = getInterleaveGroupCost(I, VF);
+ }
+ }
+ InstructionCost GatherScatterCost =
+ isLegalGatherOrScatter(I, VF)
+ ? getGatherScatterCost(I, VF) * NumAccesses
+ : InstructionCost::getInvalid();
+ InstructionCost ScalarizationCost =
+ getMemInstScalarizationCost(I, VF) * NumAccesses;
+ if (InterleaveCost <= GatherScatterCost &&
+ InterleaveCost < ScalarizationCost)
+ return {CM_Interleave, InterleaveCost};
+ if (GatherScatterCost < ScalarizationCost)
+ return {CM_GatherScatter, GatherScatterCost};
+ return {CM_Scalarize, ScalarizationCost};
+ }
+
/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
/// The cost computation for scalarized memory instruction.
- InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
+ InstructionCost getMemInstScalarizationCost(Instruction *I,
+ ElementCount VF) const;
/// The cost computation for interleaving group of memory instructions.
- InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
+ InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
/// The cost computation for Gather/Scatter instruction.
- InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
+ InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
/// The cost computation for widening instruction \p I with consecutive
/// memory access.
@@ -1517,15 +1551,6 @@ class LoopVectorizationCostModel {
/// unless necessary, e.g. when the loop isn't legal to vectorize or when
/// there is no predication.
std::function<BlockFrequencyInfo &()> GetBFI;
- /// The BlockFrequencyInfo returned from GetBFI.
- BlockFrequencyInfo *BFI = nullptr;
- /// Returns the BlockFrequencyInfo for the function if cached, otherwise
- /// fetches it via GetBFI. Avoids an indirect call to the std::function.
- BlockFrequencyInfo &getBFI() {
- if (!BFI)
- BFI = &GetBFI();
- return *BFI;
- }
const Function *TheFunction;
@@ -2400,6 +2425,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
getLoadStoreAddressSpace(I));
}
+bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
+ ElementCount VF) const {
+ assert((isa<LoadInst, StoreInst>(I)));
+ return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
+ getLoadStoreAlignment(I));
+}
+
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
ElementCount VF) {
if (!isPredicatedInst(I))
@@ -2423,7 +2455,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I),
getLoadStorePointerOperand(I));
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
- !Config.isLegalGatherOrScatter(I, VF);
+ !isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
case Instruction::SDiv:
@@ -2500,7 +2532,7 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
}
uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
- TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
+ TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) const {
if (CostKind == TTI::TCK_CodeSize)
return 1;
// If the block wasn't originally predicated then return early to avoid
@@ -2509,8 +2541,8 @@ uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
return 1;
uint64_t HeaderFreq =
- getBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
- uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
+ GetBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
+ uint64_t BBFreq = GetBFI().getBlockFreq(BB).getFrequency();
assert(HeaderFreq >= BBFreq &&
"Header has smaller block freq than dominated BB?");
return std::round((double)HeaderFreq / BBFreq);
@@ -2583,8 +2615,6 @@ LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
Instruction *I, ElementCount VF) const {
assert(isAccessInterleaved(I) && "Expecting interleaved access.");
- assert(getWideningDecision(I, VF) == CM_Unknown &&
- "Decision should not be set yet.");
auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Must have a group.");
unsigned InterleaveFactor = Group->getFactor();
@@ -3989,8 +4019,8 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return 1;
}
-bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
- ElementCount VF) {
+bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
+ Instruction *I, ElementCount VF) const {
// TODO: Cost model for emulated masked load/store is completely
// broken. This hack guides the cost model to use an artificially
// high enough value to practically disable vectorization with such
@@ -4226,7 +4256,7 @@ static const SCEV *getAddressAccessSCEV(
InstructionCost
LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(VF.isVector() &&
"Scalarization cost of instruction implies vectorization.");
if (VF.isScalable())
@@ -4351,7 +4381,7 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
const Align Alignment = getLoadStoreAlignment(I);
@@ -4374,7 +4404,7 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -4721,9 +4751,8 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
};
const InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
+ isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
+ : InstructionCost::getInvalid();
// Load: Scalar load + broadcast
// Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
@@ -4751,45 +4780,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
continue;
}
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- InstructionCost InterleaveCost = InstructionCost::getInvalid();
- unsigned NumAccesses = 1;
- if (isAccessInterleaved(&I)) {
- const auto *Group = getInterleavedAccessGroup(&I);
- assert(Group && "Fail to get an interleaved access group.");
-
- // Make one decision for the whole group.
- if (getWideningDecision(&I, VF) != CM_Unknown)
- continue;
-
- NumAccesses = Group->getNumMembers();
- if (interleavedAccessCanBeWidened(&I, VF))
- InterleaveCost = getInterleaveGroupCost(&I, VF);
- }
-
- InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF) * NumAccesses
- : InstructionCost::getInvalid();
+ // Make one decision for the whole interleave group.
+ if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
+ continue;
- InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(&I, VF) * NumAccesses;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
- // Choose better solution for the current VF,
- // write down this decision and use it during vectorization.
- InstructionCost Cost;
- InstWidening Decision;
- if (InterleaveCost <= GatherScatterCost &&
- InterleaveCost < ScalarizationCost) {
- Decision = CM_Interleave;
- Cost = InterleaveCost;
- } else if (GatherScatterCost < ScalarizationCost) {
- Decision = CM_GatherScatter;
- Cost = GatherScatterCost;
- } else {
- Decision = CM_Scalarize;
- Cost = ScalarizationCost;
- }
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
>From 23feb45c861eb0d59906fcfaf3fb403a900a014b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 07:38:24 +0100
Subject: [PATCH 2/3] [LV] Absorb uniform case as well
---
.../Transforms/Vectorize/LoopVectorize.cpp | 86 +++++++++----------
1 file changed, 39 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c7b9a815e420e..0f922bed39349 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1352,13 +1352,37 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ bool isLegalToScalarize(Instruction *I, ElementCount VF) const {
+ if (!VF.isScalable())
+ // Scalarization of fixed length vectors "just works".
+ return true;
+
+ // We have dedicated lowering for unpredicated uniform loads and
+ // stores. Note that even with tail folding we know that at least
+ // one lane is active (i.e. generalized predication is not possible
+ // here), and the logic below depends on this fact.
+ if (!foldTailByMasking())
+ return true;
+
+ // For scalable vectors, a uniform memop load is always
+ // uniform-by-parts and we know how to scalarize that.
+ if (isa<LoadInst>(I))
+ return true;
+
+ // A uniform store isn't neccessarily uniform-by-part
+ // and we can't assume scalarization.
+ auto *SI = cast<StoreInst>(I);
+ return TheLoop->isLoopInvariant(SI->getValueOperand());
+ };
+
/// Pick between interleave and gather-scatter based on cost. Returns a pair
/// of widening decision along with corresponding cost.
std::pair<InstWidening, InstructionCost>
costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ bool IsUniform = isUniformMemOp(*I, VF);
InstructionCost InterleaveCost = InstructionCost::getInvalid();
unsigned NumAccesses = 1;
- if (isAccessInterleaved(I)) {
+ if (!IsUniform && isAccessInterleaved(I)) {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -1367,13 +1391,20 @@ class LoopVectorizationCostModel {
InterleaveCost = getInterleaveGroupCost(I, VF);
}
}
+
InstructionCost GatherScatterCost =
isLegalGatherOrScatter(I, VF)
? getGatherScatterCost(I, VF) * NumAccesses
: InstructionCost::getInvalid();
+
+ // FIXME: This cost is a significant under-estimate for tail folded
+ // memory ops.
InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(I, VF) * NumAccesses;
- if (InterleaveCost <= GatherScatterCost &&
+ IsUniform ? (isLegalToScalarize(I, VF) ? getUniformMemOpCost(I, VF)
+ : InstructionCost::getInvalid())
+ : getMemInstScalarizationCost(I, VF) * NumAccesses;
+
+ if (!IsUniform && InterleaveCost <= GatherScatterCost &&
InterleaveCost < ScalarizationCost)
return {CM_Interleave, InterleaveCost};
if (GatherScatterCost < ScalarizationCost)
@@ -1403,7 +1434,7 @@ class LoopVectorizationCostModel {
/// Load: scalar load + broadcast.
/// Store: scalar store + (loop invariant value stored? 0 : extract of last
/// element)
- InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
+ InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
/// Estimate the overhead of scalarizing an instruction. This is a
/// convenience wrapper for the type-based getScalarizationOverhead API.
@@ -4346,7 +4377,7 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
InstructionCost
LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(isUniformMemOp(*I, VF));
Type *ValTy = getLoadStoreType(I);
@@ -4726,49 +4757,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (!Ptr)
continue;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
if (isUniformMemOp(I, VF)) {
- auto IsLegalToScalarize = [&]() {
- if (!VF.isScalable())
- // Scalarization of fixed length vectors "just works".
- return true;
-
- // We have dedicated lowering for unpredicated uniform loads and
- // stores. Note that even with tail folding we know that at least
- // one lane is active (i.e. generalized predication is not possible
- // here), and the logic below depends on this fact.
- if (!foldTailByMasking())
- return true;
-
- // For scalable vectors, a uniform memop load is always
- // uniform-by-parts and we know how to scalarize that.
- if (isa<LoadInst>(I))
- return true;
-
- // A uniform store isn't neccessarily uniform-by-part
- // and we can't assume scalarization.
- auto &SI = cast<StoreInst>(I);
- return TheLoop->isLoopInvariant(SI.getValueOperand());
- };
-
- const InstructionCost GatherScatterCost =
- isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
-
- // Load: Scalar load + broadcast
- // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
- // FIXME: This cost is a significant under-estimate for tail folded
- // memory ops.
- const InstructionCost ScalarizationCost =
- IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
- : InstructionCost::getInvalid();
-
// Choose better solution for the current VF, Note that Invalid
// costs compare as maximumal large. If both are invalid, we get
// scalable invalid which signals a failure and a vectorization abort.
- if (GatherScatterCost < ScalarizationCost)
- setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
- else
- setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
+ setWideningDecision(&I, VF, Decision, Cost);
continue;
}
@@ -4784,9 +4779,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
continue;
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
-
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
>From ed2087d9798ba54774062aa02c119a3a68e57ed5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 08:24:30 +0100
Subject: [PATCH 3/3] [LV] Use VectorTy in isLegalGatherOrScatter
---
.../lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp | 8 +++++---
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 7 ++++---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 64136846a80bb..f3cbdc892044f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -147,10 +147,12 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
}
bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
- Align Alignment) const {
+ Align Alignment,
+ ElementCount VF) const {
+ Type *VectorTy = toVectorTy(ScalarTy, VF);
return ForceTargetSupportsGatherScatterOps ||
- (IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
- : TTI.isLegalMaskedScatter(ScalarTy, Alignment));
+ (IsLoad ? TTI.isLegalMaskedGather(VectorTy, Alignment)
+ : TTI.isLegalMaskedScatter(VectorTy, Alignment));
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 68eebc8a1c3ad..669ff7c96b9d0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -788,9 +788,10 @@ class VFSelectionContext {
unsigned AddressSpace) const;
/// Returns true if the target machine supports a gather (if \p IsLoad)
- /// or scatter of scalar type \p ScalarTy with \p Alignment.
- bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
- Align Alignment) const;
+ /// or scatter of scalar type \p ScalarTy with \p Alignment for vectorization
+ /// factor \p VF.
+ bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy, Align Alignment,
+ ElementCount VF) const;
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0f922bed39349..f57c6ccfe229b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2460,7 +2460,7 @@ bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
ElementCount VF) const {
assert((isa<LoadInst, StoreInst>(I)));
return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
- getLoadStoreAlignment(I));
+ getLoadStoreAlignment(I), VF);
}
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
More information about the llvm-commits
mailing list