[llvm] [LV] Vectorization of compress idiom (PR #83467)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 29 11:38:14 PST 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
@llvm/pr-subscribers-llvm-transforms
Author: Kolya Panchenko (nikolaypanchenko)
<details>
<summary>Changes</summary>
Monotonic value `m` is a scalar value that has special loop carried dependency, which can be described as
```
m += step
... m ...
```
where
* `m` is a scalar variable, *`step` is a loop-invariant variable,
* the update is done under some non-uniform condition,
* use(s) is(are) done under the same or nested condition(s)
Whether `m` is used in rhs or lhs defines which special vector code needs to be generated on a use-side.
If `m` is used in lhs, the pattern is known as compress as stored data needs to be compressed according to the mask before the store
If `m` is used in rhs, the pattern is known as expand/decompress as use data needs to be expanded according to the mask
The changeset adds new descriptor for monotonic values as define above and adds initial support to vectorize unit-strided compress store.
The changeset bails out in case if monotonic value requires `expand` or `expandload` or step of it is not `1`. Future work is planed to support these scenarios.
---
Patch is 84.00 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/83467.diff
18 Files Affected:
- (modified) llvm/include/llvm/Analysis/IVDescriptors.h (+49)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+8)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+2)
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+58)
- (modified) llvm/lib/Analysis/IVDescriptors.cpp (+124)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+4)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+4)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h (+4)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+88)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+93-6)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+1)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+108-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp (+8-8)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+68)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+62)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanValue.h (+2)
- (added) llvm/test/Transforms/LoopVectorize/RISCV/compress_expand.ll (+702)
``````````diff
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 5c7b613ac48c40..877204a8b2d864 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/ValueHandle.h"
+#include "llvm/ADT/SetVector.h"
namespace llvm {
@@ -395,6 +396,54 @@ class InductionDescriptor {
SmallVector<Instruction *, 2> RedundantCasts;
};
+class MonotonicDescriptor {
+public:
+ /// This enum represents the kinds of monotonic that we support.
+ enum MonotonicKind {
+ MK_None, ///< Not a monotonic variable.
+ MK_Integer, /// < Integer monotonic variable. Step = C
+ MK_Pointer, /// < Pointer monotonic variable. Step = C
+ };
+
+public:
+ MonotonicDescriptor() = default;
+
+ Value *getStartValue() const { return StartValue; }
+ MonotonicKind getKind() const { return MK; }
+ const SCEV *getStep() const { return Step; }
+ const Instruction *getUpdateOp() const { return UpdateOp; }
+ const SetVector<PHINode *> &getPhis() const { return Phis; }
+ bool isHeaderPhi(const PHINode *Phi) const {
+ return !Phis.empty() && Phis[0] == Phi;
+ }
+
+ /// Returns true if \p Phi forms monotonic pattern within a loop \p L.
+ static MonotonicDescriptor isMonotonicPHI(PHINode *Phi, const Loop *L,
+ PredicatedScalarEvolution &PSE);
+
+ operator bool() const { return MK != MK_None; }
+
+private:
+ /// Private constructor - used by \c isMonotonicPHI
+ MonotonicDescriptor(Value *Start, MonotonicKind K, const SCEV *Step,
+ const Instruction *UpdateOp, SetVector<PHINode *> &Phis)
+ : StartValue(Start), MK(K), Step(Step), UpdateOp(UpdateOp),
+ Phis(Phis.begin(), Phis.end()) {}
+
+ /// Start value.
+ TrackingVH<Value> StartValue = nullptr;
+ /// Induction kind.
+ MonotonicKind MK = MK_None;
+ /// Step value.
+ const SCEV *Step = nullptr;
+ // Instruction that advances induction variable.
+ const Instruction *UpdateOp = nullptr;
+
+ /// All phis that are used to update the monotonic variable. It's expected
+ /// that the first PHINode is in the header BB
+ SetVector<PHINode *> Phis;
+};
+
} // end namespace llvm
#endif // LLVM_ANALYSIS_IVDESCRIPTORS_H
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 58577a6b6eb5c0..a7bdefe0d95708 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1701,6 +1701,9 @@ class TargetTransformInfo {
bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
Align Alignment) const;
+ /// \returns true if vectorization of monotonics is supported by the target.
+ bool enableMonotonicVectorization() const;
+
struct VPLegalization {
enum VPTransform {
// keep the predicating parameter
@@ -2131,6 +2134,7 @@ class TargetTransformInfo::Concept {
virtual bool supportsScalableVectors() const = 0;
virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
Align Alignment) const = 0;
+ virtual bool enableMonotonicVectorization() const = 0;
virtual VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
virtual bool hasArmWideBranch(bool Thumb) const = 0;
@@ -2874,6 +2878,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
return Impl.hasActiveVectorLength(Opcode, DataType, Alignment);
}
+ bool enableMonotonicVectorization() const override {
+ return Impl.enableMonotonicVectorization();
+ }
+
VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const override {
return Impl.getVPLegalizationStrategy(PI);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 13379cc126a40c..e77838882ee725 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -923,6 +923,8 @@ class TargetTransformInfoImplBase {
return false;
}
+ bool enableMonotonicVectorization() const { return false; }
+
TargetTransformInfo::VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const {
return TargetTransformInfo::VPLegalization(
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index a509ebf6a7e1b3..9896211ca11d83 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -257,6 +257,10 @@ class LoopVectorizationLegality {
/// induction descriptor.
using InductionList = MapVector<PHINode *, InductionDescriptor>;
+ /// MonotonicPhiList contains phi nodes that represent monotonic idiom
+ using MonotonicPhiList =
+ MapVector<const PHINode *, MonotonicDescriptor>;
+
/// RecurrenceSet contains the phi nodes that are recurrences other than
/// inductions and reductions.
using RecurrenceSet = SmallPtrSet<const PHINode *, 8>;
@@ -306,6 +310,42 @@ class LoopVectorizationLegality {
/// Returns True if V is a Phi node of an induction variable in this loop.
bool isInductionPhi(const Value *V) const;
+ /// Returns the Monotonics found in the loop
+ const MonotonicPhiList &getMonotonics() const { return MonotonicPhis; }
+
+ /// Returns the MonotonicDescriptor associated with an \p I instruction
+ /// Returns emtpy descriptor if \p I instruction is non-monotonic.
+ const MonotonicDescriptor *getMonotonicDescriptor(const Instruction *I) const {
+ for (const auto &PMD : getMonotonics()) {
+ if (const auto *Phi = dyn_cast<const PHINode>(I))
+ if (PMD.second.getPhis().contains(const_cast<PHINode *>(Phi)))
+ return &PMD.second;
+ if (PMD.second.getUpdateOp() == I)
+ return &PMD.second;
+ }
+ return nullptr;
+ }
+
+ /// Returns true if \p I instruction is a header phi of the monotonic.
+ bool isMonotonicPhi(const Instruction *I) const {
+ const auto *Phi = dyn_cast<PHINode>(I);
+ return Phi && MonotonicPhis.contains(Phi);
+ }
+
+ /// Returns true if \p V value is a header phi of the monotonic.
+ bool isMonotonicPhi(const Value *V) const {
+ const auto *I = dyn_cast<Instruction>(V);
+ return I && isMonotonicPhi(I);
+ }
+
+ /// Returns true of \p I instruction is an update instruction of the
+ /// monotonic.
+ bool isMonotonicUpdate(const Instruction *I) const {
+ return any_of(getMonotonics(), [I](const auto &PMD) {
+ return PMD.second.getUpdateOp() == I;
+ });
+ }
+
/// Returns a pointer to the induction descriptor, if \p Phi is an integer or
/// floating point induction.
const InductionDescriptor *getIntOrFpInductionDescriptor(PHINode *Phi) const;
@@ -346,6 +386,13 @@ class LoopVectorizationLegality {
/// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
+ /// Returns true if \p Ptr is depends on a monotonic value and ptr diff
+ /// between two iterations is one if monotonic value is updated
+ bool isConsecutiveMonotonicPtr(Value *Ptr) const;
+
+ /// Return true if \p Ptr computation depends on monotonic value.
+ bool ptrHasMonotonicOperand(Value *Ptr) const;
+
/// Returns true if value V is uniform across \p VF lanes, when \p VF is
/// provided, and otherwise if \p V is invariant across all loop iterations.
bool isInvariant(Value *V) const;
@@ -443,6 +490,11 @@ class LoopVectorizationLegality {
/// specific checks for outer loop vectorization.
bool canVectorizeOuterLoop();
+ /// Return true if loop vectorizer can generate correct code for that
+ /// monotonic. The method is needed to gradually enable vectorization of
+ /// monotonics.
+ bool canVectorizeMonotonic(const MonotonicDescriptor &MD);
+
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
/// \p SafePtrs is a list of addresses that are known to be legal and we know
@@ -460,6 +512,9 @@ class LoopVectorizationLegality {
void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID,
SmallPtrSetImpl<Value *> &AllowedExit);
+ /// Add MonotonicDescriptor
+ void addMonotonic(const MonotonicDescriptor &MD);
+
/// The loop that we evaluate.
Loop *TheLoop;
@@ -510,6 +565,9 @@ class LoopVectorizationLegality {
/// loop body.
SmallPtrSet<Instruction *, 4> InductionCastsToIgnore;
+ /// Holds the phis of the monotonics
+ MonotonicPhiList MonotonicPhis;
+
/// Holds the phi nodes that are fixed-order recurrences.
RecurrenceSet FixedOrderRecurrences;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 055f121e743411..9194a5622b7dc2 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1475,6 +1475,130 @@ bool InductionDescriptor::isInductionPHI(PHINode *Phi, const Loop *TheLoop,
return isInductionPHI(Phi, TheLoop, PSE.getSE(), D, AR);
}
+MonotonicDescriptor
+MonotonicDescriptor::isMonotonicPHI(PHINode *Phi, const Loop *L,
+ PredicatedScalarEvolution &PSE) {
+ // Monotonic is a special loop carried dependency which is
+ // incremented by a invariant value under some condition and used under the
+ // same or nested condition. That's different to conditional reduction, which
+ // does not allow uses at all.
+
+ // Don't allow multiple updates of the value
+ if (Phi->getNumIncomingValues() != 2)
+ return MonotonicDescriptor();
+
+ Type *Ty = Phi->getType();
+ if (!Ty->isIntegerTy() && !Ty->isPointerTy())
+ return MonotonicDescriptor();
+
+ SetVector<PHINode *> Visited;
+ Visited.insert(Phi);
+ SmallVector<PHINode *> Worklist;
+
+ for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
+ if (Phi->getIncomingBlock(I) == L->getLoopPreheader())
+ continue;
+ auto *P = dyn_cast<PHINode>(Phi->getIncomingValue(I));
+ if (!P)
+ return MonotonicDescriptor();
+ Worklist.push_back(P);
+ }
+
+ auto FindSelfUpdate = [&]() -> Instruction * {
+ Instruction *SelfUpdate = nullptr;
+ // Visit use-def chain of the Phi expecting all incoming values as phis
+ // which are used just once, i.e. within that chain.
+ while (!Worklist.empty()) {
+ PHINode *P = Worklist.pop_back_val();
+ if (Visited.contains(P))
+ continue;
+
+ Visited.insert(P);
+ // Expect all phi to be a part of the loop
+ if (!L->contains(P))
+ return nullptr;
+
+ for (unsigned I = 0, E = P->getNumIncomingValues(); I != E; ++I) {
+ Value *V = P->getIncomingValue(I);
+ if (auto *PN = dyn_cast<PHINode>(V)) {
+ Worklist.push_back(PN);
+ continue;
+ }
+ if (SelfUpdate != nullptr)
+ return nullptr;
+
+ if ((Ty->isIntegerTy() && !isa<BinaryOperator>(V)) ||
+ (Ty->isPointerTy() && !isa<GetElementPtrInst>(V)))
+ return nullptr;
+
+ SelfUpdate = cast<Instruction>(V);
+ }
+ }
+ return SelfUpdate;
+ };
+ Instruction *SelfUpdate = FindSelfUpdate();
+
+ // Expect `SelfUpdate` to bey used only once
+ // TODO: Support monotonic with a pre-increment
+ if (!SelfUpdate || SelfUpdate->getNumUses() != 1)
+ return MonotonicDescriptor();
+
+ Value *Step = nullptr;
+ if (auto *GEPUpdate = dyn_cast<GetElementPtrInst>(SelfUpdate)) {
+ if (GEPUpdate->getNumOperands() != 2)
+ return MonotonicDescriptor();
+
+ Step = GEPUpdate->getOperand(1);
+ // TODO: Re-enable update via GEP. This will require changes in VPlan to
+ // correctly print and generate updates
+ return MonotonicDescriptor();
+ }
+ auto *BO = cast<BinaryOperator>(SelfUpdate);
+ // TODO: support other than Add instruction to update monotonic variable
+ if (BO->getOpcode() != Instruction::Add)
+ return MonotonicDescriptor();
+
+ // Either `nsw` or `nuw` should be set, otherwise it's not safe to assume
+ // monotonic won't wrap.
+ if (!BO->hasNoSignedWrap() && !BO->hasNoUnsignedWrap())
+ return MonotonicDescriptor();
+ Step = BO->getOperand(0) == Phi ? BO->getOperand(1) : BO->getOperand(0);
+
+ if (!L->isLoopInvariant(Step))
+ return MonotonicDescriptor();
+
+ auto *StepSCEV = PSE.getSCEV(Step);
+ if (auto *C = dyn_cast<SCEVConstant>(StepSCEV))
+ // TODO: handle step != 1
+ if (!C->isOne())
+ return MonotonicDescriptor();
+
+ // It's important to check all uses of the Phi and make sure they are either
+ // outside of the loop.
+ // TODO: Support uses under nested predicate, which can be supported by vectorizer
+ for (User *U : Phi->users()) {
+ auto *UI = cast<Instruction>(U);
+ if (!L->contains(UI))
+ continue;
+
+ // Ignore phis that are necessary to represent self-update
+ if (auto *P = dyn_cast<PHINode>(UI))
+ if (Visited.contains(P))
+ continue;
+
+ BasicBlock *UIParent = UI->getParent();
+ if (UIParent != SelfUpdate->getParent())
+ return MonotonicDescriptor();
+ }
+
+ Value *StartValue = Phi->getIncomingValueForBlock(L->getLoopPreheader());
+ // Record all visited Phis in a vector and place Phi at the biginning to
+ // simplify future analysis
+ return MonotonicDescriptor(StartValue,
+ Ty->isPointerTy() ? MK_Pointer : MK_Integer,
+ StepSCEV, SelfUpdate, Visited);
+}
+
bool InductionDescriptor::isInductionPHI(
PHINode *Phi, const Loop *TheLoop, ScalarEvolution *SE,
InductionDescriptor &D, const SCEV *Expr,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 1f11f0d7dd620e..becc8e821fd346 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1302,6 +1302,10 @@ bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
}
+bool TargetTransformInfo::enableMonotonicVectorization() const {
+ return TTIImpl->enableMonotonicVectorization();
+}
+
TargetTransformInfo::Concept::~Concept() = default;
TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 2e4e69fb4f920f..da01d2f986b4b4 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1609,3 +1609,7 @@ bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
C2.NumIVMuls, C2.NumBaseAdds,
C2.ScaleCost, C2.ImmCost, C2.SetupCost);
}
+
+bool RISCVTTIImpl::enableMonotonicVectorization() const {
+ return ST->hasVInstructions();
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index af36e9d5d5e886..c5e6fc26605b28 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -372,6 +372,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
const TargetTransformInfo::LSRCost &C2);
+ /// \returns true if ISA supports all needed instructions to vectorize
+ /// monotonics
+ bool enableMonotonicVectorization() const;
+
bool shouldFoldTerminatingConditionAfterLSR() const {
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 37a356c43e29a4..77348826e067cf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -78,6 +78,11 @@ static cl::opt<LoopVectorizeHints::ScalableForceKind>
"Scalable vectorization is available and favored when the "
"cost is inconclusive.")));
+static cl::opt<bool>
+ EnableMonotonics("enable-monotonics", cl::init(true), cl::Hidden,
+ cl::desc("Control whether vectorization of loops with "
+ "monotonic variables is enabled"));
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -471,6 +476,36 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
return 0;
}
+bool LoopVectorizationLegality::isConsecutiveMonotonicPtr(Value *Ptr) const {
+ assert(ptrHasMonotonicOperand(Ptr) &&
+ "Pointer's computation does not use monotonic values.");
+
+ auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ assert(GEP->getNumOperands() == 2 &&
+ "GetElementPtr with more than 1 indexes is not currently supported "
+ "and should be filtered out before.");
+ Value *Monotonic = GEP->getOperand(1);
+ if (auto *Cast = dyn_cast<CastInst>(Monotonic))
+ Monotonic = Cast->getOperand(0);
+ const MonotonicDescriptor *MD =
+ getMonotonicDescriptor(cast<Instruction>(Monotonic));
+ assert(MD && "The index has no MonotonicDescriptor associated with it.");
+ const SCEVConstant *Step = dyn_cast<SCEVConstant>(MD->getStep());
+ return Step && Step->getAPInt().getZExtValue() == 1;
+}
+
+bool LoopVectorizationLegality::ptrHasMonotonicOperand(
+ Value *Ptr) const {
+ auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ if (!GEP)
+ return false;
+ return any_of(GEP->operands(), [&](Value *V) {
+ if (auto *Cast = dyn_cast<CastInst>(V))
+ return isMonotonicPhi(Cast->getOperand(0));
+ return isMonotonicPhi(V);
+ });
+}
+
bool LoopVectorizationLegality::isInvariant(Value *V) const {
return LAI->isInvariant(V);
}
@@ -678,6 +713,47 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
return Result;
}
+bool LoopVectorizationLegality::canVectorizeMonotonic(const MonotonicDescriptor &MD) {
+ Value *Monotonic = MD.getPhis().front();
+ auto IsUserInLoop = [&](User *U) -> bool {
+ auto *I = dyn_cast<Instruction>(U);
+ return I && TheLoop->contains(I);
+ };
+ auto CanIgnoreUser = [&](User *U) -> bool {
+ if (auto *PN = dyn_cast<PHINode>(U))
+ if (MD.getPhis().contains(PN))
+ return true;
+ return U == MD.getUpdateOp();
+ };
+
+ for (User *U : Monotonic->users()) {
+ if (!IsUserInLoop(U) || CanIgnoreUser(U))
+ continue;
+
+ // For now expect monotonic value to be used by by zext with a single user
+ // or GEP
+ if (U->hasOneUser() && isa<ZExtInst, SExtInst>(U))
+ U = *cast<Instruction>(U)->users().begin();
+
+ if (!isa<GetElementPtrInst>(U))
+ return false;
+
+ // All GEPs should be used as a pointer operand of a store which represents
+ // compressstore.
+ if (any_of(U->users(), [&](User *UI) {
+ if (!IsUserInLoop(UI) || CanIgnoreUser(UI))
+ return false;
+ return UI != MD.getUpdateOp() &&
+ (!isa<StoreInst>(UI) || getLoadStorePointerOperand(UI) != U);
+ })) {
+ LLVM_DEBUG(
+ dbgs() << "LV: Expand of a monotonic value is not yet supported.\n");
+ return false;
+ }
+ }
+ return true;
+}
+
void LoopVectorizationLegality::addInductionPhi(
PHINode *Phi, const InductionDescriptor &ID,
SmallPtrSetImpl<Value *> &AllowedExit) {
@@ -730,6 +806,11 @@ void LoopVectorizationLegality::addInductionPhi(
LLVM_DEBUG(dbgs() << "LV: Found an induction variable.\n");
}
+void LoopVectorizationLegality::addMonotonic(const MonotonicDescriptor &MD) {
+ for (PHINode *P : MD.getPhis())
+ MonotonicPhis[P] = MD;
+}
+
bool LoopVectorizationLegality::setupOuterLoopInductions() {
BasicBlock *Header = TheLoop->getHeader();
@@ -880,6 +961,13 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
addInductionPhi(Phi, ID, AllowedExit);
continue;
}
+ if (EnableMonotonics && TTI->enableMonotonicVectorization())
+ if (auto MD =
+ MonotonicDescriptor::isMonotonicPHI(Phi, TheLoop, PSE))
+ if (canVectorizeMonotonic(MD)) {
+ addMon...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/83467
More information about the llvm-commits
mailing list