[llvm-branch-commits] [llvm] [LoopVectorize] Support vectorization of compressing patterns in VPlan (PR #214491)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 6 06:36:40 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
RFC link: https://discourse.llvm.org/t/rfc-loop-vectorization-of-compress-store-expand-load-patterns/86442
This adds loop vectorizer support for "compressing" patterns,
for example:
```
int dst_idx = 0;
for (int i = 0; i < n; i++) {
if (cond[i])
dst[dst_idx++] = src[i];
}
```
Can be vectorized with a `llvm.masked.compressstore` as:
```
int dst_idx = 0;
for (int i = 0; i < n; i++) {
%cond = load(%cond) != 0
%src = masked.load(%src[i], %cond)
masked.compressstore(%src, %dst[dst_idx], %cond)
dst_idx += num.active.lanes(%cond)
}
```
and:
```
int src_idx = 0;
for (int i = 0; i < n; i++) {
if (cond[i])
dst[i] = src[src_idx++];
}
```
Can be vectorized with a `llvm.masked.expandload` as:
```
int src_idx = 0;
for (int i = 0; i < n; i++) {
%cond = load(%cond) != 0
%src = masked.expandload(%src[src_idx], %cond)
masked.store(%src, %dst[i], %cond)
src_idx += num.active.lanes(%cond)
}
```
This uses the new `MonotonicDescriptor` to recognize
monotonic/compressing patterns. The phis are mapped to a new
`VPMonotonicPHIRecipe`, this will map to a scalar phi. We only
allow uniform uses of monotonic phis in the loop (e.g., as the pointer
to a compressed load/store).
Compressed loads/stores are recognized with
`LoopVectorizationLegality::isCompressedPtr`. Currently, we only allow
cases where:
- The (monotonic) pointer has a stride equal to the access size
- The memory operation is predicated with the same condition as the increment
This is a continuation Sergey Kachkov's patch (#<!-- -->140723).
There are a number of changes from the initial patch:
- Expandloads/compresstores directly use `VPWidenMemIntrinsic`
- ComputeMonotonicResult is replaced with existing VP instructions
- AArch64, VPlan, and target-agnostic tests have been added
- This style of vectorization if off by default
- The switch can be flipped soon after this patch lands
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 77.89 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214491.diff
19 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+15)
- (modified) llvm/lib/Analysis/VectorUtils.cpp (+6)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+34)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp (+7)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h (+6)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+111-14)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+3-3)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+60-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (+6)
- (modified) llvm/lib/Transforms/Vectorize/VPlanHelpers.h (+4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+70-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+49)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+8)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+2-2)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll (+132)
- (added) llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll (+157)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll (+1)
- (added) llvm/test/Transforms/LoopVectorize/compress-idioms.ll (+424)
- (modified) llvm/unittests/Transforms/Vectorize/VPlanTestBase.h (+1)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 3e8db73fd79d2..c4a35ffaa0ee7 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -293,6 +293,10 @@ class LoopVectorizationLegality {
/// induction descriptor.
using InductionList = MapVector<PHINode *, InductionDescriptor>;
+ /// MonotonicPHIList saves monotonic phi variables and maps them to the
+ /// monotonic phi descriptor.
+ using MonotonicPHIList = MapVector<PHINode *, MonotonicDescriptor>;
+
/// RecurrenceSet contains the phi nodes that are recurrences other than
/// inductions and reductions.
using RecurrenceSet = SmallPtrSet<const PHINode *, 8>;
@@ -336,6 +340,11 @@ class LoopVectorizationLegality {
/// Returns the induction variables found in the loop.
const InductionList &getInductionVars() const { return Inductions; }
+ /// Returns the monotonic phi variables found in the loop.
+ const MonotonicPHIList &getMonotonicPHIs() const { return MonotonicPHIs; }
+
+ bool hasMonotonicPHIs() const { return !MonotonicPHIs.empty(); }
+
/// Return the fixed-order recurrences found in the loop.
RecurrenceSet &getFixedOrderRecurrences() { return FixedOrderRecurrences; }
@@ -389,6 +398,9 @@ class LoopVectorizationLegality {
/// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
LLVM_ABI int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
+ /// Check if memory access is compressed when vectorizing.
+ bool isCompressedPtr(Type *AccessTy, Value *Ptr, BasicBlock *BB) const;
+
/// Returns true if \p V is invariant across all loop iterations according to
/// SCEV.
LLVM_ABI bool isInvariant(Value *V) const;
@@ -695,6 +707,9 @@ class LoopVectorizationLegality {
/// variables can be pointers.
InductionList Inductions;
+ /// Holds all of the monotonic phi variables that we found in the loop.
+ MonotonicPHIList MonotonicPHIs;
+
/// Holds all the casts that participate in the update chain of the induction
/// variables, and that have been proven to be redundant (possibly under a
/// runtime guard). These casts can be ignored when creating the vectorized
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 193fb6720cf60..f501c4ee058fa 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -159,6 +159,7 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
case Intrinsic::vp_is_fpclass:
case Intrinsic::powi:
case Intrinsic::vector_extract:
+ case Intrinsic::masked_compressstore:
return (ScalarOpdIdx == 1);
case Intrinsic::smul_fix:
case Intrinsic::smul_fix_sat:
@@ -171,6 +172,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
return ScalarOpdIdx == 2 || ScalarOpdIdx == 4;
case Intrinsic::experimental_vp_strided_load:
return ScalarOpdIdx == 0 || ScalarOpdIdx == 1;
+ case Intrinsic::masked_expandload:
+ return ScalarOpdIdx == 0;
case Intrinsic::loop_dependence_war_mask:
return true;
default:
@@ -201,6 +204,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::scmp:
case Intrinsic::vector_extract:
case Intrinsic::loop_dependence_war_mask:
+ case Intrinsic::masked_expandload:
return OpdIdx == -1 || OpdIdx == 0;
case Intrinsic::modf:
case Intrinsic::sincos:
@@ -213,6 +217,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
return OpdIdx == -1 || OpdIdx == 1;
case Intrinsic::experimental_vp_strided_load:
return OpdIdx == -1 || OpdIdx == 0 || OpdIdx == 1;
+ case Intrinsic::masked_compressstore:
+ return OpdIdx == 0 || OpdIdx == 1;
default:
return OpdIdx == -1;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 9086880599231..0d7f22cea0b03 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -48,6 +48,10 @@ AllowStridedPointerIVs("lv-strided-pointer-ivs", cl::init(false), cl::Hidden,
cl::desc("Enable recognition of non-constant strided "
"pointer induction variables."));
+static cl::opt<bool> EnableMonotonicPatterns(
+ "lv-monotonic-patterns", cl::init(false), cl::Hidden,
+ cl::desc("Enable recognition of monotonic patterns."));
+
static cl::opt<bool>
HintsAllowReordering("hints-allow-reordering", cl::init(true), cl::Hidden,
cl::desc("Allow enabling loop hints to reorder "
@@ -469,6 +473,29 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
return 0;
}
+bool LoopVectorizationLegality::isCompressedPtr(Type *AccessTy, Value *Ptr,
+ BasicBlock *BB) const {
+ if (!EnableMonotonicPatterns)
+ return false;
+
+ MonotonicDescriptor Desc;
+ if (!MonotonicDescriptor::isMonotonicVal(Ptr, TheLoop, Desc, *PSE.getSE()))
+ return false;
+
+ // Check that the memory operation has the same predicate as the step.
+ // TODO: Relax these restrictions.
+ if (Desc.getPredicateEdge() !=
+ MonotonicDescriptor::Edge(BB, BB->getUniqueSuccessor()))
+ return false;
+
+ // Check if pointer step equals access size.
+ auto *Step =
+ dyn_cast<SCEVConstant>(Desc.getExpr()->getStepRecurrence(*PSE.getSE()));
+ if (!Step)
+ return false;
+ return Step->getAPInt() == BB->getDataLayout().getTypeAllocSize(AccessTy);
+}
+
bool LoopVectorizationLegality::isInvariant(Value *V) const {
return LAI->isInvariant(V);
}
@@ -883,6 +910,13 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return true;
}
+ MonotonicDescriptor MD;
+ if (EnableMonotonicPatterns &&
+ MonotonicDescriptor::isMonotonicPHI(Phi, TheLoop, MD, *PSE.getSE())) {
+ MonotonicPHIs[Phi] = MD;
+ return true;
+ }
+
if (RecurrenceDescriptor::isFixedOrderRecurrence(Phi, TheLoop, DT)) {
FixedOrderRecurrences.insert(Phi);
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index fd7fd2e011a83..d5c711f1fab42 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -157,6 +157,13 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
(SI && TTI.isLegalMaskedScatter(Ty, Align));
}
+bool VFSelectionContext::isLegalExpandLoadOrCompressStore(
+ bool IsLoad, Type *ScalarTy, Align Alignment) const {
+ return ForceTargetSupportsMaskedMemoryOps ||
+ (IsLoad ? TTI.isLegalMaskedExpandLoad(ScalarTy, Alignment)
+ : TTI.isLegalMaskedCompressStore(ScalarTy, Alignment));
+}
+
bool VFSelectionContext::supportsScalableVectors() const {
return TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors ||
VectorizerParams::VectorizationFactor.isScalable();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fa317f290022e..c9a06b4b2117e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -791,6 +791,12 @@ class VFSelectionContext {
/// or scatter operation.
bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
+ /// Returns true if the target machine supports a masked expand load (if \p
+ /// IsLoad) or masked compress store of scalar type \p ScalarTy with \p
+ /// Alignment.
+ bool isLegalExpandLoadOrCompressStore(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const;
+
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
/// InLoopReductionImmediateChains is filled with each in-loop reduction
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3eb6d0530da6d..24174ae275d2e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -878,6 +878,7 @@ class LoopVectorizationCostModel {
CM_Widen_Reverse, // For consecutive accesses with stride -1.
CM_Interleave,
CM_GatherScatter,
+ CM_Compressed,
CM_Scalarize,
/// A widening decision that has been invalidated after replacing the
/// corresponding recipe during VPlan transforms.
@@ -1063,6 +1064,10 @@ class LoopVectorizationCostModel {
/// consecutive or part of an interleave group.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
+ /// Returns true if the target machine supports a masked expand load or masked
+ /// compress store for \p I's data type and alignment.
+ bool isLegalExpandLoadOrCompressStore(Instruction *I) const;
+
/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) const {
return InterleaveInfo.isInterleaved(Instr);
@@ -2400,6 +2405,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
getLoadStoreAddressSpace(I));
}
+bool LoopVectorizationCostModel::isLegalExpandLoadOrCompressStore(
+ Instruction *I) const {
+ assert(isa<LoadInst>(I) || isa<StoreInst>(I));
+ return Config.isLegalExpandLoadOrCompressStore(
+ isa<LoadInst>(I), getLoadStoreType(I), getLoadStoreAlignment(I));
+}
+
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
ElementCount VF) {
if (!isPredicatedInst(I))
@@ -2420,9 +2432,13 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
}
case Instruction::Load:
case Instruction::Store: {
- bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I),
- getLoadStorePointerOperand(I));
+ Type *ScalarTy = getLoadStoreType(I);
+ Value *Ptr = getLoadStorePointerOperand(I);
+ bool IsConsecutive = Legal->isConsecutivePtr(ScalarTy, Ptr);
+ bool IsCompressed =
+ !IsConsecutive && Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
+ !(IsCompressed && isLegalExpandLoadOrCompressStore(I)) &&
!Config.isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
@@ -2661,8 +2677,9 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
auto *Ptr = getLoadStorePointerOperand(I);
auto *ScalarTy = getLoadStoreType(I);
- // In order to be widened, the pointer should be consecutive, first of all.
- int Stride = Legal->isConsecutivePtr(ScalarTy, Ptr);
+ // In order to be widened, the pointer should be consecutive or compressed.
+ bool Compressed = Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
+ int Stride = Compressed ? 1 : Legal->isConsecutivePtr(ScalarTy, Ptr);
if (!Stride)
return std::nullopt;
@@ -2677,6 +2694,8 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
if (hasIrregularType(ScalarTy, DL))
return std::nullopt;
+ if (Compressed)
+ return CM_Compressed;
return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
}
@@ -2770,9 +2789,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (IsUniformMemOpUse(I))
return true;
- return (WideningDecision == CM_Widen ||
- WideningDecision == CM_Widen_Reverse ||
- WideningDecision == CM_Interleave);
+ return (
+ WideningDecision == CM_Widen || WideningDecision == CM_Widen_Reverse ||
+ WideningDecision == CM_Interleave || WideningDecision == CM_Compressed);
};
// Returns true if Ptr is the pointer operand of a memory access instruction
@@ -2917,6 +2936,38 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
AddToWorklistIfAllowed(IndUpdate);
}
+ // Handle monotonic phis (similarly to induction vars).
+ for (const auto &MonotonicPHI : Legal->getMonotonicPHIs()) {
+ auto *Phi = MonotonicPHI.first;
+ auto *PhiUpdate = cast<Instruction>(Phi->getIncomingValueForBlock(Latch));
+ const auto &Desc = MonotonicPHI.second;
+
+ auto UniformPhi = all_of(Phi->users(), [&](User *U) -> bool {
+ auto *I = cast<Instruction>(U);
+ if (I == Desc.getStepInst())
+ return true;
+ if (auto *PN = dyn_cast<PHINode>(I); PN && Desc.getChain().contains(PN))
+ return true;
+ return !TheLoop->contains(I) || Worklist.count(I) ||
+ IsVectorizedMemAccessUse(I, Phi);
+ });
+ if (!UniformPhi)
+ continue;
+
+ auto UniformPhiUpdate = all_of(PhiUpdate->users(), [&](User *U) -> bool {
+ auto *I = cast<Instruction>(U);
+ if (I == Phi)
+ return true;
+ return !TheLoop->contains(I) || Worklist.count(I) ||
+ IsVectorizedMemAccessUse(I, Phi);
+ });
+ if (!UniformPhiUpdate)
+ continue;
+
+ AddToWorklistIfAllowed(Phi);
+ AddToWorklistIfAllowed(PhiUpdate);
+ }
+
Uniforms[VF].insert_range(Worklist);
}
@@ -3291,6 +3342,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPExpandSCEVSC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPBranchOnMaskSC:
+ case VPRecipeBase::VPMonotonicPHISC:
continue;
case VPRecipeBase::VPReductionSC:
case VPRecipeBase::VPActiveLaneMaskPHISC:
@@ -3676,6 +3728,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (Plan.hasEarlyExit())
return 1;
+ // Monotonic vars don't support interleaving.
+ if (Legal->hasMonotonicPHIs())
+ return 1;
+
const bool HasReductions =
any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
IsaPred<VPReductionPHIRecipe>);
@@ -4036,8 +4092,10 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
// of the instruction.
// 2. Scalable VF, as that would lead to invalid scalarization costs.
// 3. Emulated masked memrefs, if a hacked cost is needed.
+ // 4. Compressed loads/stores (which do not support scalarization)
if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
!useEmulatedMaskMemRefHack(&I, VF) &&
+ getWideningDecision(&I, VF) != CM_Compressed &&
computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
for (const auto &[I, IC] : ScalarCosts)
ScalarCostsVF.insert({I, IC});
@@ -4287,8 +4345,9 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *I, ElementCount VF, InstWidening Kind) {
- assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
- "Expected a consecutive widening decision");
+ assert(
+ (Kind == CM_Widen || Kind == CM_Widen_Reverse || Kind == CM_Compressed) &&
+ "Expected a consecutive widening decision");
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(I);
@@ -4296,9 +4355,14 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
const Align Alignment = getLoadStoreAlignment(I);
InstructionCost Cost = 0;
if (isMaskRequired(I)) {
- unsigned IID = I->getOpcode() == Instruction::Load
- ? Intrinsic::masked_load
- : Intrinsic::masked_store;
+ Intrinsic::ID LoadIID = Intrinsic::masked_load;
+ Intrinsic::ID StoreIID = Intrinsic::masked_store;
+ if (Kind == CM_Compressed) {
+ LoadIID = Intrinsic::masked_expandload;
+ StoreIID = Intrinsic::masked_compressstore;
+ }
+
+ unsigned IID = I->getOpcode() == Instruction::Load ? LoadIID : StoreIID;
Cost += TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
Config.CostKind);
@@ -5276,6 +5340,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI::CastContextHint::Reversed;
case LoopVectorizationCostModel::CM_Unknown:
llvm_unreachable("Instr did not go through cost modelling?");
+ case LoopVectorizationCostModel::CM_Compressed:
+ // TODO: Add Compressed hint (not needed for any targets yet).
case LoopVectorizationCostModel::CM_InvalidatedDecision:
return TTI::CastContextHint::None;
}
@@ -5636,6 +5702,11 @@ bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
(VF.isVector() && CM.isProfitableToScalarize(I, VF));
}
+bool VPCostContext::isUniformAfterVectorization(Instruction *I,
+ ElementCount VF) const {
+ return CM.isUniformAfterVectorization(I, VF);
+}
+
bool VPCostContext::isMaskRequired(Instruction *I) const {
return CM.isMaskRequired(I);
}
@@ -6225,8 +6296,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
LoopVectorizationCostModel::InstWidening Decision =
CM.getWideningDecision(I, Range.Start);
bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
+ bool Compressed = Decision == LoopVectorizationCostModel::CM_Compressed;
bool Consecutive =
- Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
+ Reverse || Compressed || Decision == LoopVectorizationCostModel::CM_Widen;
VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
@@ -6241,6 +6313,13 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
+ Type *LoadTy = Load->getType();
+
+ if (Compressed)
+ return Builder.createWidenMemIntrinsic(
+ Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(LoadTy)},
+ LoadTy, Load->getAlign(), *VPI, Load->getDebugLoc());
+
auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI,
Load->getDebugLoc());
if (Reverse)
@@ -6251,6 +6330,12 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
StoreInst *Store = cast<StoreInst>(I);
VPValue *StoredVal = VPI->getOperand(0);
+ if (Compressed)
+ return Builder.createWidenMemIntrinsic(
+ Intrinsic::masked_compressstore, {StoredVal, Ptr, Mask},
+ StoredVal->getScalarType(), Store->getAlign(), *VPI,
+ Store->getDebugLoc());
+
if (Reverse)
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
Store->getDebugLoc());
@@ -6565,7 +6650,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// and in-loop reductions are empty since legality doesn't detect them.
if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
*OrigLoop, VPDT, Legal->getInductionVars(),
- Legal->getReductionVars(),
+ Legal->getReductionVars(), Legal->getMonotonicPHIs(),
Legal->getFixedOrderRecurrences(),
Config.getInLoopReductions(), Hints.allowReordering())) {
return nullptr;
@@ -6834,6 +6919,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// bring the VPlan to its final state.
// ---------------------------------------------------------------------------
+ RUN_VPLAN_PASS(VPlanTransforms::adjustMonotonicPhiBackedgeUsers, *Plan,
+ HeaderVPBB, PSE);
+
addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
// Optimize FindIV reductions to use sentinel-based approach when possible.
@@ -8120,6 +8208,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
unsigned SelectedIC = std::max(IC, UserIC);
+
+ if (LVL.hasMonotonicPHIs() && Sel...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214491
More information about the llvm-branch-commits
mailing list