[llvm] [VPlan] Make widen-gather-scatter decision in VPlan (PR #215862)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 00:50:57 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/215862
>From 5a20a69636655f24fa452635a32e3e846fd616a5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 12 Aug 2026 18:14:21 +0100
Subject: [PATCH 1/5] [LV] Factor costInterleaveGatherScatter (NFC)
The motivation for factoring out a costInterleaveGatherScatter that
compares the cost of interleaving versus that of a gather-scatter is for
re-use in a follow-up doing VPlan-based gather-scatter-widening.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 16 +--
.../Vectorize/LoopVectorizationPlanner.h | 7 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 127 +++++++++---------
3 files changed, 70 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index ff9b9171d8c8c..64136846a80bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -146,19 +146,11 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
: TTI.isLegalMaskedStore(ScalarTy, Alignment, AddressSpace));
}
-bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
- ElementCount VF) const {
- bool LI = isa<LoadInst>(V);
- bool SI = isa<StoreInst>(V);
- if (!LI && !SI)
- return false;
- auto *Ty = getLoadStoreType(V);
- Align Align = getLoadStoreAlignment(V);
- if (VF.isVector())
- Ty = VectorType::get(Ty, VF);
+bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const {
return ForceTargetSupportsGatherScatterOps ||
- (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
- (SI && TTI.isLegalMaskedScatter(Ty, Align));
+ (IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
+ : TTI.isLegalMaskedScatter(ScalarTy, Alignment));
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..68eebc8a1c3ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -787,9 +787,10 @@ class VFSelectionContext {
bool isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy, Align Alignment,
unsigned AddressSpace) const;
- /// Returns true if the target machine can represent \p V as a masked gather
- /// or scatter operation.
- bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
+ /// Returns true if the target machine supports a gather (if \p IsLoad)
+ /// or scatter of scalar type \p ScalarTy with \p Alignment.
+ bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const;
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d892ff2a0fa2b..c7b9a815e420e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1034,11 +1034,11 @@ class LoopVectorizationCostModel {
/// every iteration of the loop header.
inline uint64_t
getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
- const BasicBlock *BB);
+ const BasicBlock *BB) const;
/// Returns true if an artificially high cost for emulated masked memrefs
/// should be used.
- bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
+ bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
/// Return the costs for our two available strategies for lowering a
/// div/rem operation which requires speculating at least one lane.
@@ -1063,6 +1063,10 @@ class LoopVectorizationCostModel {
/// consecutive or part of an interleave group.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
+ /// Returns true if the target machine supports gather or scatter for \p I's
+ /// data type and alignment.
+ bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
+
/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) const {
return InterleaveInfo.isInterleaved(Instr);
@@ -1348,17 +1352,47 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ /// Pick between interleave and gather-scatter based on cost. Returns a pair
+ /// of widening decision along with corresponding cost.
+ std::pair<InstWidening, InstructionCost>
+ costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ InstructionCost InterleaveCost = InstructionCost::getInvalid();
+ unsigned NumAccesses = 1;
+ if (isAccessInterleaved(I)) {
+ const auto *Group = getInterleavedAccessGroup(I);
+ assert(Group && "Fail to get an interleaved access group.");
+
+ if (interleavedAccessCanBeWidened(I, VF)) {
+ NumAccesses = Group->getNumMembers();
+ InterleaveCost = getInterleaveGroupCost(I, VF);
+ }
+ }
+ InstructionCost GatherScatterCost =
+ isLegalGatherOrScatter(I, VF)
+ ? getGatherScatterCost(I, VF) * NumAccesses
+ : InstructionCost::getInvalid();
+ InstructionCost ScalarizationCost =
+ getMemInstScalarizationCost(I, VF) * NumAccesses;
+ if (InterleaveCost <= GatherScatterCost &&
+ InterleaveCost < ScalarizationCost)
+ return {CM_Interleave, InterleaveCost};
+ if (GatherScatterCost < ScalarizationCost)
+ return {CM_GatherScatter, GatherScatterCost};
+ return {CM_Scalarize, ScalarizationCost};
+ }
+
/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
/// The cost computation for scalarized memory instruction.
- InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
+ InstructionCost getMemInstScalarizationCost(Instruction *I,
+ ElementCount VF) const;
/// The cost computation for interleaving group of memory instructions.
- InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
+ InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
/// The cost computation for Gather/Scatter instruction.
- InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
+ InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
/// The cost computation for widening instruction \p I with consecutive
/// memory access.
@@ -1517,15 +1551,6 @@ class LoopVectorizationCostModel {
/// unless necessary, e.g. when the loop isn't legal to vectorize or when
/// there is no predication.
std::function<BlockFrequencyInfo &()> GetBFI;
- /// The BlockFrequencyInfo returned from GetBFI.
- BlockFrequencyInfo *BFI = nullptr;
- /// Returns the BlockFrequencyInfo for the function if cached, otherwise
- /// fetches it via GetBFI. Avoids an indirect call to the std::function.
- BlockFrequencyInfo &getBFI() {
- if (!BFI)
- BFI = &GetBFI();
- return *BFI;
- }
const Function *TheFunction;
@@ -2400,6 +2425,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
getLoadStoreAddressSpace(I));
}
+bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
+ ElementCount VF) const {
+ assert((isa<LoadInst, StoreInst>(I)));
+ return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
+ getLoadStoreAlignment(I));
+}
+
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
ElementCount VF) {
if (!isPredicatedInst(I))
@@ -2423,7 +2455,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I),
getLoadStorePointerOperand(I));
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
- !Config.isLegalGatherOrScatter(I, VF);
+ !isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
case Instruction::SDiv:
@@ -2500,7 +2532,7 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
}
uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
- TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
+ TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) const {
if (CostKind == TTI::TCK_CodeSize)
return 1;
// If the block wasn't originally predicated then return early to avoid
@@ -2509,8 +2541,8 @@ uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
return 1;
uint64_t HeaderFreq =
- getBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
- uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
+ GetBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
+ uint64_t BBFreq = GetBFI().getBlockFreq(BB).getFrequency();
assert(HeaderFreq >= BBFreq &&
"Header has smaller block freq than dominated BB?");
return std::round((double)HeaderFreq / BBFreq);
@@ -2583,8 +2615,6 @@ LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
Instruction *I, ElementCount VF) const {
assert(isAccessInterleaved(I) && "Expecting interleaved access.");
- assert(getWideningDecision(I, VF) == CM_Unknown &&
- "Decision should not be set yet.");
auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Must have a group.");
unsigned InterleaveFactor = Group->getFactor();
@@ -3989,8 +4019,8 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return 1;
}
-bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
- ElementCount VF) {
+bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
+ Instruction *I, ElementCount VF) const {
// TODO: Cost model for emulated masked load/store is completely
// broken. This hack guides the cost model to use an artificially
// high enough value to practically disable vectorization with such
@@ -4226,7 +4256,7 @@ static const SCEV *getAddressAccessSCEV(
InstructionCost
LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(VF.isVector() &&
"Scalarization cost of instruction implies vectorization.");
if (VF.isScalable())
@@ -4351,7 +4381,7 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
const Align Alignment = getLoadStoreAlignment(I);
@@ -4374,7 +4404,7 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -4721,9 +4751,8 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
};
const InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
+ isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
+ : InstructionCost::getInvalid();
// Load: Scalar load + broadcast
// Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
@@ -4751,45 +4780,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
continue;
}
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- InstructionCost InterleaveCost = InstructionCost::getInvalid();
- unsigned NumAccesses = 1;
- if (isAccessInterleaved(&I)) {
- const auto *Group = getInterleavedAccessGroup(&I);
- assert(Group && "Fail to get an interleaved access group.");
-
- // Make one decision for the whole group.
- if (getWideningDecision(&I, VF) != CM_Unknown)
- continue;
-
- NumAccesses = Group->getNumMembers();
- if (interleavedAccessCanBeWidened(&I, VF))
- InterleaveCost = getInterleaveGroupCost(&I, VF);
- }
-
- InstructionCost GatherScatterCost =
- Config.isLegalGatherOrScatter(&I, VF)
- ? getGatherScatterCost(&I, VF) * NumAccesses
- : InstructionCost::getInvalid();
+ // Make one decision for the whole interleave group.
+ if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
+ continue;
- InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(&I, VF) * NumAccesses;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
- // Choose better solution for the current VF,
- // write down this decision and use it during vectorization.
- InstructionCost Cost;
- InstWidening Decision;
- if (InterleaveCost <= GatherScatterCost &&
- InterleaveCost < ScalarizationCost) {
- Decision = CM_Interleave;
- Cost = InterleaveCost;
- } else if (GatherScatterCost < ScalarizationCost) {
- Decision = CM_GatherScatter;
- Cost = GatherScatterCost;
- } else {
- Decision = CM_Scalarize;
- Cost = ScalarizationCost;
- }
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
>From 23feb45c861eb0d59906fcfaf3fb403a900a014b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 07:38:24 +0100
Subject: [PATCH 2/5] [LV] Absorb uniform case as well
---
.../Transforms/Vectorize/LoopVectorize.cpp | 86 +++++++++----------
1 file changed, 39 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c7b9a815e420e..0f922bed39349 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1352,13 +1352,37 @@ class LoopVectorizationCostModel {
: std::nullopt);
}
+ bool isLegalToScalarize(Instruction *I, ElementCount VF) const {
+ if (!VF.isScalable())
+ // Scalarization of fixed length vectors "just works".
+ return true;
+
+ // We have dedicated lowering for unpredicated uniform loads and
+ // stores. Note that even with tail folding we know that at least
+ // one lane is active (i.e. generalized predication is not possible
+ // here), and the logic below depends on this fact.
+ if (!foldTailByMasking())
+ return true;
+
+ // For scalable vectors, a uniform memop load is always
+ // uniform-by-parts and we know how to scalarize that.
+ if (isa<LoadInst>(I))
+ return true;
+
+ // A uniform store isn't neccessarily uniform-by-part
+ // and we can't assume scalarization.
+ auto *SI = cast<StoreInst>(I);
+ return TheLoop->isLoopInvariant(SI->getValueOperand());
+ };
+
/// Pick between interleave and gather-scatter based on cost. Returns a pair
/// of widening decision along with corresponding cost.
std::pair<InstWidening, InstructionCost>
costInterleaveGatherScatter(Instruction *I, ElementCount VF) const {
+ bool IsUniform = isUniformMemOp(*I, VF);
InstructionCost InterleaveCost = InstructionCost::getInvalid();
unsigned NumAccesses = 1;
- if (isAccessInterleaved(I)) {
+ if (!IsUniform && isAccessInterleaved(I)) {
const auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.");
@@ -1367,13 +1391,20 @@ class LoopVectorizationCostModel {
InterleaveCost = getInterleaveGroupCost(I, VF);
}
}
+
InstructionCost GatherScatterCost =
isLegalGatherOrScatter(I, VF)
? getGatherScatterCost(I, VF) * NumAccesses
: InstructionCost::getInvalid();
+
+ // FIXME: This cost is a significant under-estimate for tail folded
+ // memory ops.
InstructionCost ScalarizationCost =
- getMemInstScalarizationCost(I, VF) * NumAccesses;
- if (InterleaveCost <= GatherScatterCost &&
+ IsUniform ? (isLegalToScalarize(I, VF) ? getUniformMemOpCost(I, VF)
+ : InstructionCost::getInvalid())
+ : getMemInstScalarizationCost(I, VF) * NumAccesses;
+
+ if (!IsUniform && InterleaveCost <= GatherScatterCost &&
InterleaveCost < ScalarizationCost)
return {CM_Interleave, InterleaveCost};
if (GatherScatterCost < ScalarizationCost)
@@ -1403,7 +1434,7 @@ class LoopVectorizationCostModel {
/// Load: scalar load + broadcast.
/// Store: scalar store + (loop invariant value stored? 0 : extract of last
/// element)
- InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
+ InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
/// Estimate the overhead of scalarizing an instruction. This is a
/// convenience wrapper for the type-based getScalarizationOverhead API.
@@ -4346,7 +4377,7 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
InstructionCost
LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
- ElementCount VF) {
+ ElementCount VF) const {
assert(isUniformMemOp(*I, VF));
Type *ValTy = getLoadStoreType(I);
@@ -4726,49 +4757,13 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (!Ptr)
continue;
+ // Choose between Interleaving, Gather/Scatter or Scalarization.
+ auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
if (isUniformMemOp(I, VF)) {
- auto IsLegalToScalarize = [&]() {
- if (!VF.isScalable())
- // Scalarization of fixed length vectors "just works".
- return true;
-
- // We have dedicated lowering for unpredicated uniform loads and
- // stores. Note that even with tail folding we know that at least
- // one lane is active (i.e. generalized predication is not possible
- // here), and the logic below depends on this fact.
- if (!foldTailByMasking())
- return true;
-
- // For scalable vectors, a uniform memop load is always
- // uniform-by-parts and we know how to scalarize that.
- if (isa<LoadInst>(I))
- return true;
-
- // A uniform store isn't neccessarily uniform-by-part
- // and we can't assume scalarization.
- auto &SI = cast<StoreInst>(I);
- return TheLoop->isLoopInvariant(SI.getValueOperand());
- };
-
- const InstructionCost GatherScatterCost =
- isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
- : InstructionCost::getInvalid();
-
- // Load: Scalar load + broadcast
- // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
- // FIXME: This cost is a significant under-estimate for tail folded
- // memory ops.
- const InstructionCost ScalarizationCost =
- IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
- : InstructionCost::getInvalid();
-
// Choose better solution for the current VF, Note that Invalid
// costs compare as maximumal large. If both are invalid, we get
// scalable invalid which signals a failure and a vectorization abort.
- if (GatherScatterCost < ScalarizationCost)
- setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
- else
- setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
+ setWideningDecision(&I, VF, Decision, Cost);
continue;
}
@@ -4784,9 +4779,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (isAccessInterleaved(&I) && getWideningDecision(&I, VF) != CM_Unknown)
continue;
- // Choose between Interleaving, Gather/Scatter or Scalarization.
- auto [Decision, Cost] = costInterleaveGatherScatter(&I, VF);
-
// If the instructions belongs to an interleave group, the whole group
// receives the same decision. The whole group receives the cost, but
// the cost will actually be assigned to one instruction.
>From ed2087d9798ba54774062aa02c119a3a68e57ed5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 08:24:30 +0100
Subject: [PATCH 3/5] [LV] Use VectorTy in isLegalGatherOrScatter
---
.../lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp | 8 +++++---
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 7 ++++---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 64136846a80bb..f3cbdc892044f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -147,10 +147,12 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
}
bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
- Align Alignment) const {
+ Align Alignment,
+ ElementCount VF) const {
+ Type *VectorTy = toVectorTy(ScalarTy, VF);
return ForceTargetSupportsGatherScatterOps ||
- (IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
- : TTI.isLegalMaskedScatter(ScalarTy, Alignment));
+ (IsLoad ? TTI.isLegalMaskedGather(VectorTy, Alignment)
+ : TTI.isLegalMaskedScatter(VectorTy, Alignment));
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 68eebc8a1c3ad..669ff7c96b9d0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -788,9 +788,10 @@ class VFSelectionContext {
unsigned AddressSpace) const;
/// Returns true if the target machine supports a gather (if \p IsLoad)
- /// or scatter of scalar type \p ScalarTy with \p Alignment.
- bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
- Align Alignment) const;
+ /// or scatter of scalar type \p ScalarTy with \p Alignment for vectorization
+ /// factor \p VF.
+ bool isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy, Align Alignment,
+ ElementCount VF) const;
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0f922bed39349..f57c6ccfe229b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2460,7 +2460,7 @@ bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
ElementCount VF) const {
assert((isa<LoadInst, StoreInst>(I)));
return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
- getLoadStoreAlignment(I));
+ getLoadStoreAlignment(I), VF);
}
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
>From c391093e940d0381447d128bf794b7c3458776ba Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 13 Aug 2026 08:13:24 +0100
Subject: [PATCH 4/5] [LV] Pre-commit vplan-memory-op-decisions tests
---
.../AArch64/vplan-memory-op-decisions.ll | 250 +++++++++++++++++-
1 file changed, 249 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index 0fc28f9261713..5e3b57c5cce9e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-masked-memory-ops -disable-output -vplan-print-after=widenConsecutiveMemOps %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops -disable-output -vplan-print-after=widenConsecutiveMemOps %s 2>&1 | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -232,6 +232,59 @@ exit:
ret void
}
+; A forward (stride 2) gather-scatter.
+define void @gather_scatter(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'gather_scatter'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<512> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<2>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
+; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %lv = load i32, ptr %gep.a, align 4
+ %add = add i32 %lv, 1
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 2
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
; A reverse (stride -1) consecutive load and store.
define void @reverse_load_store(ptr noalias %a, ptr noalias %b) {
; CHECK-LABEL: VPlan for loop in 'reverse_load_store'
@@ -289,6 +342,59 @@ exit:
ret void
}
+; A reverse (stride -2) gather-scatter.
+define void @reverse_gather_scatter(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'reverse_gather_scatter'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<511> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<1023>, ir<-2>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
+; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<-2>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<0>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1023, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %lv = load i32, ptr %gep.a, align 4
+ %add = add i32 %lv, 1
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %add, ptr %gep.b, align 4
+ %iv.next = add nsw i64 %iv, -2
+ %ec = icmp eq i64 %iv.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
; The load address %gep.a is indexed by %prev, a first-order recurrence.
define void @consecutive_load_with_first_order_recurrence_address(ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'consecutive_load_with_first_order_recurrence_address'
@@ -424,6 +530,78 @@ exit:
ret void
}
+; A consecutive load and store guarded by a condition.
+define void @cond_gather_scatter(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'cond_gather_scatter'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + ((-2 + %n) /u 2))<nuw>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<2>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.cond> = getelementptr inbounds ir<%cond>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%c> = load ir<%gep.cond>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp sgt ir<%c>, ir<0>
+; CHECK-NEXT: Successor(s): then
+; CHECK-EMPTY:
+; CHECK-NEXT: then:
+; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>, ir<%cmp>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.cond = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.cond, align 4
+ %cmp = icmp sgt i32 %c, 0
+ br i1 %cmp, label %then, label %latch
+
+then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %lv = load i32, ptr %gep.a, align 4
+ %add = add i32 %lv, 1
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %add, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 2
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
; A reverse (stride -1) consecutive load and store guarded by a condition. The
; mask must be reversed as well, to match the reversed access order.
define void @cond_reverse_load_store(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
@@ -503,3 +681,73 @@ latch:
exit:
ret void
}
+
+define void @cond_reverse_gather_scatter(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
+; CHECK-LABEL: VPlan for loop in 'cond_reverse_gather_scatter'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<511> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<1023>, ir<-2>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.cond> = getelementptr inbounds ir<%cond>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%c> = load ir<%gep.cond>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp sgt ir<%c>, ir<0>
+; CHECK-NEXT: Successor(s): then
+; CHECK-EMPTY:
+; CHECK-NEXT: then:
+; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>, ir<%cmp>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<-2>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<0>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1023, %entry ], [ %iv.next, %latch ]
+ %gep.cond = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.cond, align 4
+ %cmp = icmp sgt i32 %c, 0
+ br i1 %cmp, label %then, label %latch
+
+then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %lv = load i32, ptr %gep.a, align 4
+ %add = add i32 %lv, 1
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %add, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nsw i64 %iv, -2
+ %ec = icmp eq i64 %iv.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 5d4ce4e540dc484e08cd3befce973386080b62b6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 12 Aug 2026 14:55:09 +0100
Subject: [PATCH 5/5] [VPlan] Make widen-gather-scatter decision in VPlan
Port the widening of non-consecutive mem ops to
makeMemOpWideningDecisions, making it handle gathers and scatters,
matching legacy behvaior. No functional change is intended.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 +++++
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 4 +++
.../Transforms/Vectorize/VPlanTransforms.cpp | 36 ++++++++++++-------
.../AArch64/vplan-memory-op-decisions.ll | 10 +++---
.../VPlan/vplan-print-before-after-all.ll | 2 +-
5 files changed, 43 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f57c6ccfe229b..eedddd8a68bea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1319,6 +1319,10 @@ class LoopVectorizationCostModel {
return FS != ForcedScalars.end() && FS->second.contains(I);
}
+ bool isGatherScatterProfitable(Instruction *I, ElementCount VF) const {
+ return costInterleaveGatherScatter(I, VF).first == CM_GatherScatter;
+ }
+
private:
unsigned NumPredStores = 0;
@@ -5625,6 +5629,11 @@ bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
(VF.isVector() && CM.isProfitableToScalarize(I, VF));
}
+bool VPCostContext::willGatherScatter(Instruction *I, ElementCount VF) const {
+ return !willBeScalarized(I, VF) &&
+ CM.isGatherScatterProfitable(I, VF);
+}
+
bool VPCostContext::isMaskRequired(Instruction *I) const {
return CM.isMaskRequired(I);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 1c987abc649c8..b5baa5bb84ebf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -363,6 +363,10 @@ struct VPCostContext {
/// Returns true if \p I is known to be scalarized at \p VF.
bool willBeScalarized(Instruction *I, ElementCount VF) const;
+ /// Returns true if \p I is known to be profitable to gather-scatter, as
+ /// opposed to scalarizing or interleaving, for \p VF.
+ bool willGatherScatter(Instruction *I, ElementCount VF) const;
+
/// Forwards to LoopVectorizationCostModel::isMaskRequired.
bool isMaskRequired(Instruction *I) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5f125aecb005a..6fff45c75e49e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5479,10 +5479,11 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
});
}
- // Widen unit-stride consecutive accesses, matching the legacy CM. Both
- // forward (stride +1) and reverse (stride -1) accesses are handled.
+ // Widen unit-stride consecutive accesses, matching the legacy CM. Consecutive
+ // forward (stride +1) and reverse (stride -1) accesses, as well as
+ // non-consecutive accesses (gather/scatter) are handled.
VPlanTransforms::runPass(
- "widenConsecutiveMemOps", ProcessSubset, Plan, [&](VPInstruction *VPI) {
+ "widenMemOps", ProcessSubset, Plan, [&](VPInstruction *VPI) {
Instruction *I = VPI->getUnderlyingInstr();
bool IsLoad = VPI->getOpcode() == Instruction::Load;
VPValue *Ptr = VPI->getOperand(!IsLoad);
@@ -5490,8 +5491,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
IsLoad ? VPI->getScalarType() : VPI->getOperand(0)->getScalarType();
std::optional<int64_t> Stride =
getConstantStride(Ptr, ScalarTy, CostCtx.PSE, CostCtx.L);
- if (Stride != 1 && Stride != -1)
+ if (!Stride)
return false;
+ bool Consecutive = Stride == 1 || Stride == -1;
bool Reverse = Stride == -1;
// A predicated access can only be widened (rather than scalarized) if
@@ -5502,10 +5504,20 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
IsLoad, ScalarTy, getLoadStoreAlignment(I),
getLoadStoreAddressSpace(I)))
return false;
+ if (!Consecutive &&
+ !LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ return CostCtx.willGatherScatter(I, VF) &&
+ CostCtx.Config.isLegalGatherOrScatter(
+ IsLoad, ScalarTy, getLoadStoreAlignment(I), VF);
+ },
+ Range))
+ return false;
VPBuilder Builder(VPI);
- VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
- Ptr, ScalarTy, Reverse, VPI->getDebugLoc());
+ if (Consecutive)
+ Ptr = Builder.createConsecutiveVectorPointer(Ptr, ScalarTy, Reverse,
+ VPI->getDebugLoc());
VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
// Reverse the mask so it matches the reversed access order.
@@ -5514,9 +5526,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPI->getDebugLoc());
if (IsLoad) {
- VPSingleDefRecipe *Load = Builder.createWidenLoad(
- *cast<LoadInst>(I), VectorPtr, Mask,
- /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
+ VPSingleDefRecipe *Load =
+ Builder.createWidenLoad(*cast<LoadInst>(I), Ptr, Mask,
+ Consecutive, *VPI, VPI->getDebugLoc());
// Reverse the loaded values back into program order.
if (Reverse)
Load = Builder.createNaryOp(VPInstruction::Reverse, Load,
@@ -5530,9 +5542,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
VPI->getDebugLoc());
- auto *StoreR = Builder.createWidenStore(
- *cast<StoreInst>(I), VectorPtr, StoredVal, Mask,
- /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
+ auto *StoreR =
+ Builder.createWidenStore(*cast<StoreInst>(I), Ptr, StoredVal, Mask,
+ Consecutive, *VPI, VPI->getDebugLoc());
return ReplaceWith(VPI, StoreR);
});
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index 5e3b57c5cce9e..4ea15fb4b7042 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops -disable-output -vplan-print-after=widenConsecutiveMemOps %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops -disable-output -vplan-print-after=widenMemOps %s 2>&1 | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -256,7 +256,7 @@ define void @gather_scatter(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%add>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<2>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -366,7 +366,7 @@ define void @reverse_gather_scatter(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%add>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<-2>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -558,7 +558,7 @@ define void @cond_gather_scatter(ptr noalias %a, ptr noalias %b, ptr noalias %co
; CHECK-EMPTY:
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%lv> = load ir<%gep.a>, ir<%cmp>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>, ir<%cmp>
@@ -708,7 +708,7 @@ define void @cond_reverse_gather_scatter(ptr noalias %a, ptr noalias %b, ptr noa
; CHECK-EMPTY:
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%lv> = load ir<%gep.a>, ir<%cmp>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>, ir<%cmp>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a7271524f0191..d722a90ca549e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -20,7 +20,7 @@
; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] lowerMemoryIdioms
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] scalarizeMemOpsWithIrregularTypes
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenConsecutiveMemOps
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenMemOps
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] delegateMemOpWideningToLegacyCM
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions
More information about the llvm-commits
mailing list