[llvm-branch-commits] [llvm] [LoopVectorize] Support vectorization of compressing patterns (PR #214491)
Benjamin Maxwell via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 13 02:00:37 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/214491
>From 7d9bb82c61f982c336fd2e2ea65841f1dcbe9f9a Mon Sep 17 00:00:00 2001
From: Sergey Kachkov <sergey.kachkov at syntacore.com>
Date: Wed, 5 Aug 2026 18:04:38 +0000
Subject: [PATCH 1/7] [LoopVectorize] Support vectorization of compressing
patterns in VPlan
RFC link: https://discourse.llvm.org/t/rfc-loop-vectorization-of-compress-store-expand-load-patterns/86442
This adds loop vectorizer support for "compressing" patterns,
for example:
```
int dst_idx = 0;
for (int i = 0; i < n; i++) {
if (cond[i])
dst[dst_idx++] = src[i];
}
```
Can be vectorized with a `llvm.masked.compressstore` as:
```
int dst_idx = 0;
for (int i = 0; i < n; i++) {
%cond = load(%cond) != 0
%src = masked.load(%src[i], %cond)
masked.compressstore(%src, %dst[dst_idx], %cond)
dst_idx += num.active.lanes(%cond)
}
```
and:
```
int src_idx = 0;
for (int i = 0; i < n; i++) {
if (cond[i])
dst[i] = src[src_idx++];
}
```
Can be vectorized with a `llvm.masked.expandload` as:
```
int src_idx = 0;
for (int i = 0; i < n; i++) {
%cond = load(%cond) != 0
%src = masked.expandload(%src[src_idx], %cond)
masked.store(%src, %dst[i], %cond)
src_idx += num.active.lanes(%cond)
}
```
This uses the new `MonotonicDescriptor` to recognize
monotonic/compressing patterns. The phis are mapped to a new
`VPMonotonicPHIRecipe`, this will map to a scalar phi. We only
allow uniform uses of monotonic phis in the loop (e.g., as the pointer
to a compressed load/store).
Compressed loads/stores are recognized with
`LoopVectorizationLegality::isCompressedPtr`. Currently, we only allow
cases where:
- The (monotonic) pointer has a stride equal to the access size
- The memory operation is predicated with the same condition as the increment
This is a continuation Sergey Kachkov's patch (#140723).
There are a number of changes from the initial patch:
- Expandloads/compresstores directly use `VPWidenMemIntrinsic`
- ComputeMonotonicResult is replaced with existing VP instructions
- AArch64, VPlan, and target-agnostic tests have been added
- This style of vectorization if off by default
- The switch can be flipped soon after this patch lands
---
llvm/include/llvm/Analysis/VectorUtils.h | 15 +
.../Vectorize/LoopVectorizationLegality.h | 15 +
llvm/lib/Analysis/VectorUtils.cpp | 45 ++
llvm/lib/IR/IntrinsicInst.cpp | 2 +
.../Vectorize/LoopVectorizationLegality.cpp | 34 ++
.../Vectorize/LoopVectorizationPlanner.cpp | 7 +
.../Vectorize/LoopVectorizationPlanner.h | 6 +
.../Transforms/Vectorize/LoopVectorize.cpp | 125 +++++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 64 ++-
.../Vectorize/VPlanConstruction.cpp | 6 +
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 4 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 37 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 49 ++
.../Transforms/Vectorize/VPlanTransforms.h | 8 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
.../LoopVectorize/AArch64/compress-idioms.ll | 132 ++++++
.../LoopVectorize/VPlan/compress-idioms.ll | 157 +++++++
.../VPlan/vplan-print-before-after-all.ll | 1 +
.../LoopVectorize/compress-idioms.ll | 424 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTestBase.h | 1 +
21 files changed, 1116 insertions(+), 26 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/compress-idioms.ll
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index f70ae3ff57f88..37af2fbb6e2fe 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -165,6 +165,21 @@ LLVM_ABI bool
isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx,
const TargetTransformInfo *TTI);
+/// Returns the argument index of the pointer parameter for the vector memory
+/// intrinsic \p ID, or `std::nullopt` the intrinsic does not have a pointer
+/// operand.
+LLVM_ABI std::optional<unsigned>
+getVectorMemoryIntrinsicPointerArgIdx(Intrinsic::ID ID);
+
+/// Returns the argument index of the data value of the vector store intrinsic
+/// \p ID, or `std::nullopt` if the intrinsic does not have a data operand.
+LLVM_ABI std::optional<unsigned>
+getVectorStoreIntrinsicDataArgIdx(Intrinsic::ID ID);
+
+/// Returns the argument index of the mask for the vector intrinsic \p ID, or
+/// `std::nullopt` if the intrinsic does not have a mask operand.
+LLVM_ABI std::optional<unsigned> getVectorIntrinsicMaskArgIdx(Intrinsic::ID ID);
+
/// Identifies if the vector form of the intrinsic that returns a struct is
/// overloaded at the struct element index \p RetIdx. /// \p TTI is used to
/// consider target specific intrinsics, if no target specific intrinsics
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 7b8b27c6541e1..f3ba591429851 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -287,6 +287,10 @@ class LoopVectorizationLegality {
/// induction descriptor.
using InductionList = MapVector<PHINode *, InductionDescriptor>;
+ /// MonotonicPHIList saves monotonic phi variables and maps them to the
+ /// monotonic phi descriptor.
+ using MonotonicPHIList = MapVector<PHINode *, MonotonicDescriptor>;
+
/// RecurrenceSet contains the phi nodes that are recurrences other than
/// inductions and reductions.
using RecurrenceSet = SmallPtrSet<const PHINode *, 8>;
@@ -330,6 +334,11 @@ class LoopVectorizationLegality {
/// Returns the induction variables found in the loop.
const InductionList &getInductionVars() const { return Inductions; }
+ /// Returns the monotonic phi variables found in the loop.
+ const MonotonicPHIList &getMonotonicPHIs() const { return MonotonicPHIs; }
+
+ bool hasMonotonicPHIs() const { return !MonotonicPHIs.empty(); }
+
/// Return the fixed-order recurrences found in the loop.
RecurrenceSet &getFixedOrderRecurrences() { return FixedOrderRecurrences; }
@@ -383,6 +392,9 @@ class LoopVectorizationLegality {
/// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
LLVM_ABI int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
+ /// Check if memory access is compressed when vectorizing.
+ bool isCompressedPtr(Type *AccessTy, Value *Ptr, BasicBlock *BB) const;
+
/// Returns true if \p V is invariant across all loop iterations according to
/// SCEV.
LLVM_ABI bool isInvariant(Value *V) const;
@@ -689,6 +701,9 @@ class LoopVectorizationLegality {
/// variables can be pointers.
InductionList Inductions;
+ /// Holds all of the monotonic phi variables that we found in the loop.
+ MonotonicPHIList MonotonicPHIs;
+
/// Holds all the casts that participate in the update chain of the induction
/// variables, and that have been proven to be redundant (possibly under a
/// runtime guard). These casts can be ignored when creating the vectorized
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index a33f17bf00755..fab7321675a42 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -159,6 +159,7 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
case Intrinsic::vp_is_fpclass:
case Intrinsic::powi:
case Intrinsic::vector_extract:
+ case Intrinsic::masked_compressstore:
return (ScalarOpdIdx == 1);
case Intrinsic::smul_fix:
case Intrinsic::smul_fix_sat:
@@ -173,6 +174,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
return ScalarOpdIdx == 0 || ScalarOpdIdx == 1;
case Intrinsic::experimental_vp_strided_store:
return ScalarOpdIdx == 1 || ScalarOpdIdx == 2;
+ case Intrinsic::masked_expandload:
+ return ScalarOpdIdx == 0;
case Intrinsic::loop_dependence_war_mask:
return true;
default:
@@ -203,6 +206,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::scmp:
case Intrinsic::vector_extract:
case Intrinsic::loop_dependence_war_mask:
+ case Intrinsic::masked_expandload:
return OpdIdx == -1 || OpdIdx == 0;
case Intrinsic::modf:
case Intrinsic::sincos:
@@ -217,11 +221,52 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
return OpdIdx == -1 || OpdIdx == 0 || OpdIdx == 1;
case Intrinsic::experimental_vp_strided_store:
return OpdIdx == 0 || OpdIdx == 1 || OpdIdx == 2;
+ case Intrinsic::masked_compressstore:
+ return OpdIdx == 0 || OpdIdx == 1;
default:
return OpdIdx == -1;
}
}
+std::optional<unsigned>
+llvm::getVectorMemoryIntrinsicPointerArgIdx(Intrinsic::ID ID) {
+ if (auto PtrPos = VPIntrinsic::getMemoryPointerParamPos(ID))
+ return PtrPos;
+ switch (ID) {
+ case Intrinsic::masked_compressstore:
+ return 1;
+ case Intrinsic::masked_expandload:
+ return 0;
+ default:
+ return std::nullopt;
+ }
+}
+
+std::optional<unsigned>
+llvm::getVectorStoreIntrinsicDataArgIdx(Intrinsic::ID ID) {
+ if (auto DataPos = VPIntrinsic::getMemoryDataParamPos(ID))
+ return DataPos;
+ switch (ID) {
+ case Intrinsic::masked_expandload:
+ return 2;
+ default:
+ return std::nullopt;
+ }
+}
+
+std::optional<unsigned> llvm::getVectorIntrinsicMaskArgIdx(Intrinsic::ID ID) {
+ if (auto MaskPos = VPIntrinsic::getMaskParamPos(ID))
+ return MaskPos;
+ switch (ID) {
+ case Intrinsic::masked_compressstore:
+ return 2;
+ case Intrinsic::masked_expandload:
+ return 1;
+ default:
+ return std::nullopt;
+ }
+}
+
bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(
Intrinsic::ID ID, int RetIdx, const TargetTransformInfo *TTI) {
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index eb964d566f294..604b2ec88e77d 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -440,10 +440,12 @@ VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
switch (VPID) {
default:
return std::nullopt;
+ case Intrinsic::masked_compressstore:
case Intrinsic::vp_store:
case Intrinsic::vp_scatter:
case Intrinsic::experimental_vp_strided_store:
return 1;
+ case Intrinsic::masked_expandload:
case Intrinsic::vp_load:
case Intrinsic::vp_load_ff:
case Intrinsic::vp_gather:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 6a5419dfbaccf..7189d345f6d6c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -48,6 +48,10 @@ AllowStridedPointerIVs("lv-strided-pointer-ivs", cl::init(false), cl::Hidden,
cl::desc("Enable recognition of non-constant strided "
"pointer induction variables."));
+static cl::opt<bool> EnableMonotonicPatterns(
+ "lv-monotonic-patterns", cl::init(false), cl::Hidden,
+ cl::desc("Enable recognition of monotonic patterns."));
+
static cl::opt<bool>
HintsAllowReordering("hints-allow-reordering", cl::init(true), cl::Hidden,
cl::desc("Allow enabling loop hints to reorder "
@@ -473,6 +477,29 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
return Stride;
}
+bool LoopVectorizationLegality::isCompressedPtr(Type *AccessTy, Value *Ptr,
+ BasicBlock *BB) const {
+ if (!EnableMonotonicPatterns)
+ return false;
+
+ MonotonicDescriptor Desc;
+ if (!MonotonicDescriptor::isMonotonicVal(Ptr, TheLoop, Desc, *PSE.getSE()))
+ return false;
+
+ // Check that the memory operation has the same predicate as the step.
+ // TODO: Relax these restrictions.
+ if (Desc.getPredicateEdge() !=
+ MonotonicDescriptor::Edge(BB, BB->getUniqueSuccessor()))
+ return false;
+
+ // Check if pointer step equals access size.
+ auto *Step =
+ dyn_cast<SCEVConstant>(Desc.getExpr()->getStepRecurrence(*PSE.getSE()));
+ if (!Step)
+ return false;
+ return Step->getAPInt() == BB->getDataLayout().getTypeAllocSize(AccessTy);
+}
+
bool LoopVectorizationLegality::isInvariant(Value *V) const {
return LAI->isInvariant(V);
}
@@ -905,6 +932,13 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return true;
}
+ MonotonicDescriptor MD;
+ if (EnableMonotonicPatterns &&
+ MonotonicDescriptor::isMonotonicPHI(Phi, TheLoop, MD, *PSE.getSE())) {
+ MonotonicPHIs[Phi] = MD;
+ return true;
+ }
+
if (RecurrenceDescriptor::isFixedOrderRecurrence(Phi, TheLoop, DT)) {
FixedOrderRecurrences.insert(Phi);
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index ff9b9171d8c8c..cfe84561d9132 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -161,6 +161,13 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
(SI && TTI.isLegalMaskedScatter(Ty, Align));
}
+bool VFSelectionContext::isLegalExpandLoadOrCompressStore(
+ bool IsLoad, Type *ScalarTy, Align Alignment) const {
+ return ForceTargetSupportsMaskedMemoryOps ||
+ (IsLoad ? TTI.isLegalMaskedExpandLoad(ScalarTy, Alignment)
+ : TTI.isLegalMaskedCompressStore(ScalarTy, Alignment));
+}
+
bool VFSelectionContext::supportsScalableVectors() const {
return TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors ||
VectorizerParams::VectorizationFactor.isScalable();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..3b50d6bf08af6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -791,6 +791,12 @@ class VFSelectionContext {
/// or scatter operation.
bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
+ /// Returns true if the target machine supports a masked expand load (if \p
+ /// IsLoad) or masked compress store of scalar type \p ScalarTy with \p
+ /// Alignment.
+ bool isLegalExpandLoadOrCompressStore(bool IsLoad, Type *ScalarTy,
+ Align Alignment) const;
+
/// Split reductions into those that happen in the loop, and those that
/// happen outside. In-loop reductions are collected into InLoopReductions.
/// InLoopReductionImmediateChains is filled with each in-loop reduction
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0647644515b94..fb74f9a7bca26 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -873,6 +873,7 @@ class LoopVectorizationCostModel {
CM_Widen_Reverse, // For consecutive accesses with stride -1.
CM_Interleave,
CM_GatherScatter,
+ CM_Compressed,
CM_Scalarize,
/// A widening decision that has been invalidated after replacing the
/// corresponding recipe during VPlan transforms.
@@ -1058,6 +1059,10 @@ class LoopVectorizationCostModel {
/// consecutive or part of an interleave group.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
+ /// Returns true if the target machine supports a masked expand load or masked
+ /// compress store for \p I's data type and alignment.
+ bool isLegalExpandLoadOrCompressStore(Instruction *I) const;
+
/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) const {
return InterleaveInfo.isInterleaved(Instr);
@@ -2386,6 +2391,13 @@ bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
getLoadStoreAddressSpace(I));
}
+bool LoopVectorizationCostModel::isLegalExpandLoadOrCompressStore(
+ Instruction *I) const {
+ assert(isa<LoadInst>(I) || isa<StoreInst>(I));
+ return Config.isLegalExpandLoadOrCompressStore(
+ isa<LoadInst>(I), getLoadStoreType(I), getLoadStoreAlignment(I));
+}
+
bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
ElementCount VF) {
if (!isPredicatedInst(I))
@@ -2406,9 +2418,13 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
}
case Instruction::Load:
case Instruction::Store: {
- bool IsConsecutive = Legal->isConsecutivePtr(getLoadStoreType(I),
- getLoadStorePointerOperand(I));
+ Type *ScalarTy = getLoadStoreType(I);
+ Value *Ptr = getLoadStorePointerOperand(I);
+ bool IsConsecutive = Legal->isConsecutivePtr(ScalarTy, Ptr);
+ bool IsCompressed =
+ !IsConsecutive && Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
+ !(IsCompressed && isLegalExpandLoadOrCompressStore(I)) &&
!Config.isLegalGatherOrScatter(I, VF);
}
case Instruction::UDiv:
@@ -2647,8 +2663,9 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
auto *Ptr = getLoadStorePointerOperand(I);
auto *ScalarTy = getLoadStoreType(I);
- // In order to be widened, the pointer should be consecutive, first of all.
- int Stride = Legal->isConsecutivePtr(ScalarTy, Ptr);
+ // In order to be widened, the pointer should be consecutive or compressed.
+ bool Compressed = Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
+ int Stride = Compressed ? 1 : Legal->isConsecutivePtr(ScalarTy, Ptr);
if (!Stride)
return std::nullopt;
@@ -2663,6 +2680,8 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
if (hasIrregularType(ScalarTy, DL))
return std::nullopt;
+ if (Compressed)
+ return CM_Compressed;
return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
}
@@ -2756,9 +2775,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (IsUniformMemOpUse(I))
return true;
- return (WideningDecision == CM_Widen ||
- WideningDecision == CM_Widen_Reverse ||
- WideningDecision == CM_Interleave);
+ return (
+ WideningDecision == CM_Widen || WideningDecision == CM_Widen_Reverse ||
+ WideningDecision == CM_Interleave || WideningDecision == CM_Compressed);
};
// Returns true if Ptr is the pointer operand of a memory access instruction
@@ -2903,6 +2922,38 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
AddToWorklistIfAllowed(IndUpdate);
}
+ // Handle monotonic phis (similarly to induction vars).
+ for (const auto &MonotonicPHI : Legal->getMonotonicPHIs()) {
+ auto *Phi = MonotonicPHI.first;
+ auto *PhiUpdate = cast<Instruction>(Phi->getIncomingValueForBlock(Latch));
+ const auto &Desc = MonotonicPHI.second;
+
+ auto UniformPhi = all_of(Phi->users(), [&](User *U) -> bool {
+ auto *I = cast<Instruction>(U);
+ if (I == Desc.getStepInst())
+ return true;
+ if (auto *PN = dyn_cast<PHINode>(I); PN && Desc.getChain().contains(PN))
+ return true;
+ return !TheLoop->contains(I) || Worklist.count(I) ||
+ IsVectorizedMemAccessUse(I, Phi);
+ });
+ if (!UniformPhi)
+ continue;
+
+ auto UniformPhiUpdate = all_of(PhiUpdate->users(), [&](User *U) -> bool {
+ auto *I = cast<Instruction>(U);
+ if (I == Phi)
+ return true;
+ return !TheLoop->contains(I) || Worklist.count(I) ||
+ IsVectorizedMemAccessUse(I, Phi);
+ });
+ if (!UniformPhiUpdate)
+ continue;
+
+ AddToWorklistIfAllowed(Phi);
+ AddToWorklistIfAllowed(PhiUpdate);
+ }
+
Uniforms[VF].insert_range(Worklist);
}
@@ -3277,6 +3328,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPExpandSCEVSC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPBranchOnMaskSC:
+ case VPRecipeBase::VPMonotonicPHISC:
continue;
case VPRecipeBase::VPReductionSC:
case VPRecipeBase::VPActiveLaneMaskPHISC:
@@ -3660,6 +3712,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (Plan.hasEarlyExit())
return 1;
+ // Monotonic vars don't support interleaving.
+ if (Legal->hasMonotonicPHIs())
+ return 1;
+
const bool HasReductions =
any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
IsaPred<VPReductionPHIRecipe>);
@@ -4020,8 +4076,10 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
// of the instruction.
// 2. Scalable VF, as that would lead to invalid scalarization costs.
// 3. Emulated masked memrefs, if a hacked cost is needed.
+ // 4. Compressed loads/stores (which do not support scalarization)
if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
!useEmulatedMaskMemRefHack(&I, VF) &&
+ getWideningDecision(&I, VF) != CM_Compressed &&
computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
for (const auto &[I, IC] : ScalarCosts)
ScalarCostsVF.insert({I, IC});
@@ -4271,8 +4329,9 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *I, ElementCount VF, InstWidening Kind) {
- assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
- "Expected a consecutive widening decision");
+ assert(
+ (Kind == CM_Widen || Kind == CM_Widen_Reverse || Kind == CM_Compressed) &&
+ "Expected a consecutive widening decision");
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(I);
@@ -4280,9 +4339,14 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
const Align Alignment = getLoadStoreAlignment(I);
InstructionCost Cost = 0;
if (isMaskRequired(I)) {
- unsigned IID = I->getOpcode() == Instruction::Load
- ? Intrinsic::masked_load
- : Intrinsic::masked_store;
+ Intrinsic::ID LoadIID = Intrinsic::masked_load;
+ Intrinsic::ID StoreIID = Intrinsic::masked_store;
+ if (Kind == CM_Compressed) {
+ LoadIID = Intrinsic::masked_expandload;
+ StoreIID = Intrinsic::masked_compressstore;
+ }
+
+ unsigned IID = I->getOpcode() == Instruction::Load ? LoadIID : StoreIID;
Cost += TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
Config.CostKind);
@@ -5260,6 +5324,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI::CastContextHint::Reversed;
case LoopVectorizationCostModel::CM_Unknown:
llvm_unreachable("Instr did not go through cost modelling?");
+ case LoopVectorizationCostModel::CM_Compressed:
+ // TODO: Add Compressed hint (not needed for any targets yet).
case LoopVectorizationCostModel::CM_InvalidatedDecision:
return TTI::CastContextHint::None;
}
@@ -5620,6 +5686,11 @@ bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
(VF.isVector() && CM.isProfitableToScalarize(I, VF));
}
+bool VPCostContext::isUniformAfterVectorization(Instruction *I,
+ ElementCount VF) const {
+ return CM.isUniformAfterVectorization(I, VF);
+}
+
bool VPCostContext::isMaskRequired(Instruction *I) const {
return CM.isMaskRequired(I);
}
@@ -6216,8 +6287,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
LoopVectorizationCostModel::InstWidening Decision =
CM.getWideningDecision(I, Range.Start);
bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
+ bool Compressed = Decision == LoopVectorizationCostModel::CM_Compressed;
bool Consecutive =
- Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
+ Reverse || Compressed || Decision == LoopVectorizationCostModel::CM_Widen;
VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
@@ -6232,6 +6304,13 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
+ Type *LoadTy = Load->getType();
+
+ if (Compressed)
+ return Builder.createWidenMemIntrinsic(
+ Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(LoadTy)},
+ LoadTy, Load->getAlign(), *VPI, Load->getDebugLoc());
+
auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI,
Load->getDebugLoc());
if (Reverse)
@@ -6242,6 +6321,12 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
StoreInst *Store = cast<StoreInst>(I);
VPValue *StoredVal = VPI->getOperand(0);
+ if (Compressed)
+ return Builder.createWidenMemIntrinsic(
+ Intrinsic::masked_compressstore, {StoredVal, Ptr, Mask},
+ StoredVal->getScalarType(), Store->getAlign(), *VPI,
+ Store->getDebugLoc());
+
if (Reverse)
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
Store->getDebugLoc());
@@ -6556,7 +6641,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// and in-loop reductions are empty since legality doesn't detect them.
if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
*OrigLoop, VPDT, Legal->getInductionVars(),
- Legal->getReductionVars(),
+ Legal->getReductionVars(), Legal->getMonotonicPHIs(),
Legal->getFixedOrderRecurrences(),
Config.getInLoopReductions(), Hints.allowReordering())) {
return nullptr;
@@ -6825,6 +6910,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// bring the VPlan to its final state.
// ---------------------------------------------------------------------------
+ RUN_VPLAN_PASS(VPlanTransforms::adjustMonotonicPhiBackedgeUsers, *Plan,
+ HeaderVPBB, PSE);
+
addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
// Optimize FindIV reductions to use sentinel-based approach when possible.
@@ -8113,6 +8201,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
unsigned SelectedIC = std::max(IC, UserIC);
+
+ if (LVL.hasMonotonicPHIs() && SelectedIC > 1) {
+ reportVectorizationFailure(
+ "Interleaving of loop with monotonic vars",
+ "Interleaving of loops with monotonic vars is not supported",
+ "CantInterleaveWithMonotonicVars", ORE, L);
+ return false;
+ }
+
// Optimistically generate runtime checks if they are needed. Drop them if
// they turn out to not be profitable.
if (VF.Width.isVector() || SelectedIC > 1) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f077932cf7217..1573063d108d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -403,9 +403,9 @@ void VPTransformState::fixupHeaderPhis() {
for (VPRecipeBase &R : Header->phis()) {
auto *PhiR = cast<VPSingleDefRecipe>(&R);
- bool NeedsScalar =
- isa<VPPhi>(PhiR) || (isa<VPReductionPHIRecipe>(PhiR) &&
- cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
+ bool NeedsScalar = isa<VPPhi>(PhiR) || isa<VPMonotonicPHIRecipe>(PhiR) ||
+ (isa<VPReductionPHIRecipe>(PhiR) &&
+ cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
Value *Phi = get(PhiR, NeedsScalar);
Value *Val = get(PhiR->getOperand(1), NeedsScalar);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b2e87a8f4f52d..bc0d924b91e3f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -462,12 +462,13 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPWidenIntOrFpInductionSC,
VPWidenPointerInductionSC,
VPReductionPHISC,
+ VPMonotonicPHISC,
// END: SubclassID for recipes that inherit VPHeaderPHIRecipe
// END: Phi-like recipes
VPFirstPHISC = VPWidenPHISC,
VPFirstHeaderPHISC = VPCurrentIterationPHISC,
- VPLastHeaderPHISC = VPReductionPHISC,
- VPLastPHISC = VPReductionPHISC,
+ VPLastHeaderPHISC = VPMonotonicPHISC,
+ VPLastPHISC = VPMonotonicPHISC,
};
VPRecipeBase(VPRecipeTy SC, ArrayRef<VPValue *> Operands,
@@ -660,6 +661,7 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase,
case VPRecipeBase::VPReductionPHISC:
case VPRecipeBase::VPWidenLoadEVLSC:
case VPRecipeBase::VPWidenLoadSC:
+ case VPRecipeBase::VPMonotonicPHISC:
return true;
case VPRecipeBase::VPBranchOnMaskSC:
case VPRecipeBase::VPInterleaveEVLSC:
@@ -2075,7 +2077,9 @@ class VPWidenMemIntrinsicRecipe final : public VPWidenIntrinsicRecipe {
DL),
Alignment(Alignment) {
assert((VectorIntrinsicID == Intrinsic::experimental_vp_strided_load ||
- VectorIntrinsicID == Intrinsic::experimental_vp_strided_store) &&
+ VectorIntrinsicID == Intrinsic::experimental_vp_strided_store ||
+ VectorIntrinsicID == Intrinsic::masked_compressstore ||
+ VectorIntrinsicID == Intrinsic::masked_expandload) &&
"Unexpected intrinsic");
}
@@ -2953,6 +2957,57 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
#endif
};
+/// A recipe for handling monotonic phis. The start value is the first operand
+/// of the recipe and the incoming value from the backedge is the second
+/// operand.
+class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe {
+ MonotonicDescriptor Desc;
+
+public:
+ VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc,
+ VPValue &Start, VPValue &BackedgeValue)
+ : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, Phi, &Start),
+ Desc(Desc) {
+
+ addOperand(&BackedgeValue);
+ }
+
+ ~VPMonotonicPHIRecipe() override = default;
+
+ VPMonotonicPHIRecipe *clone() override {
+ auto *R =
+ new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()), Desc,
+ *getStartValue(), *getBackedgeValue());
+ return R;
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPMonotonicPHISC)
+
+ static inline bool classof(const VPHeaderPHIRecipe *R) {
+ return R->getVPRecipeID() == VPRecipeBase::VPMonotonicPHISC;
+ }
+
+ void execute(VPTransformState &State) override;
+
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ const MonotonicDescriptor &getDescriptor() const { return Desc; }
+
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool usesFirstLaneOnly(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ return true;
+ }
+};
+
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
/// instructions.
class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
@@ -4341,7 +4396,8 @@ struct CastInfoMixinImpl
template <>
struct CastInfo<VPPhiAccessors, VPRecipeBase *>
: vpdetail::CastInfoMixinImpl<VPPhiAccessors, VPPhi, VPIRPhi,
- VPWidenPHIRecipe, VPHeaderPHIRecipe> {};
+ VPWidenPHIRecipe, VPHeaderPHIRecipe,
+ VPMonotonicPHIRecipe> {};
template <>
struct CastInfo<VPPhiAccessors, const VPRecipeBase *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4c09e10fe6ea0..aea63707ad5a2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -927,6 +927,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
const VPDominatorTree &VPDT,
const MapVector<PHINode *, InductionDescriptor> &Inductions,
const MapVector<PHINode *, RecurrenceDescriptor> &Reductions,
+ const MapVector<PHINode *, MonotonicDescriptor> &MonotonicPHIs,
const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering) {
// Retrieve the header manually from the intial plain-CFG VPlan.
@@ -959,6 +960,11 @@ bool VPlanTransforms::createHeaderPhiRecipes(
Plan, PSE, OrigLoop,
PhiR->getDebugLoc());
+ auto MonotonicIt = MonotonicPHIs.find(Phi);
+ if (MonotonicIt != MonotonicPHIs.end())
+ return new VPMonotonicPHIRecipe(Phi, MonotonicIt->second, *Start,
+ *BackedgeValue);
+
assert(Reductions.contains(Phi) && "only reductions are expected now");
const RecurrenceDescriptor &RdxDesc = Reductions.lookup(Phi);
assert(RdxDesc.getRecurrenceStartValue() ==
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 1c987abc649c8..06ff7d4c3cc63 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -360,6 +360,10 @@ struct VPCostContext {
/// Forwards to LoopVectorizationCostModel::getPredBlockCostDivisor.
uint64_t getPredBlockCostDivisor(BasicBlock *BB) const;
+ /// Returns true if \p I is known to be uniform after vectorization.
+ /// Forwards to LoopVectorizationCostModel::isUniformAfterVectorization.
+ bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const;
+
/// Returns true if \p I is known to be scalarized at \p VF.
bool willBeScalarized(Instruction *I, ElementCount VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 53ab77fbd28b1..53b58af954782 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1231,6 +1231,7 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked
: TTI::CastContextHint::Normal;
}
+
const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
if (WidenMemoryRecipe == nullptr)
return TTI::CastContextHint::None;
@@ -1471,6 +1472,12 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
VectorTy, /*Mask=*/{}, Ctx.CostKind,
/*Index=*/0);
}
+ case VPInstruction::NumActiveLanes: {
+ Type *ElementTy = getOperand(0)->getScalarType();
+ auto *VectorTy = cast<VectorType>(toVectorTy(ElementTy, VF));
+ return Ctx.TTI.getArithmeticReductionCost(Instruction::Add, VectorTy,
+ std::nullopt, Ctx.CostKind);
+ }
case VPInstruction::ExtractLastLane: {
// Add on the cost of extracting the element.
auto *VecTy = toVectorTy(getOperand(0)->getScalarType(), VF);
@@ -2414,7 +2421,7 @@ void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPWidenMemIntrinsicRecipe::execute(VPTransformState &State) {
CallInst *MemI = createVectorCall(State);
- auto PtrPos = VPIntrinsic::getMemoryPointerParamPos(getVectorIntrinsicID());
+ auto PtrPos = getVectorMemoryIntrinsicPointerArgIdx(getVectorIntrinsicID());
assert(PtrPos && "Expected a memory intrinsic with a valid pointer position");
MemI->addParamAttr(
*PtrPos, Attribute::getWithAlignment(MemI->getContext(), Alignment));
@@ -2434,13 +2441,13 @@ InstructionCost
VPWidenMemIntrinsicRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
Type *DataTy;
- if (auto DataPos = VPIntrinsic::getMemoryDataParamPos(getVectorIntrinsicID()))
+ if (auto DataPos = getVectorStoreIntrinsicDataArgIdx(getVectorIntrinsicID()))
DataTy = getOperand(*DataPos)->getScalarType();
else
DataTy = getScalarType();
assert(!DataTy->isVoidTy() && "Expected a non-void data type");
Type *Ty = toVectorTy(DataTy, VF);
- auto MaskPos = VPIntrinsic::getMaskParamPos(getVectorIntrinsicID());
+ auto MaskPos = getVectorIntrinsicMaskArgIdx(getVectorIntrinsicID());
assert(MaskPos && "Expected a memory intrinsic with a valid mask position");
return computeMemIntrinsicCost(getVectorIntrinsicID(), Ty,
!match(getOperand(*MaskPos), m_True()),
@@ -5012,6 +5019,30 @@ bool VPBlendRecipe::usesFirstLaneOnly(const VPValue *Op) const {
return vputils::onlyFirstLaneUsed(this);
}
+void VPMonotonicPHIRecipe::execute(VPTransformState &State) {
+ executePhiRecipe(this, *this, State, /*IsScalar=*/true, "monotonic.iv");
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPMonotonicPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "MONOTONIC-PHI ";
+
+ printAsOperand(O, SlotTracker);
+ O << " = phi ";
+ printOperands(O, SlotTracker);
+}
+#endif
+
+InstructionCost VPMonotonicPHIRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ auto *Phi = cast<PHINode>(getUnderlyingValue());
+ // The value of a monotonic phi must be uniform across the VF.
+ if (!Ctx.isUniformAfterVectorization(Phi, VF))
+ return InstructionCost::getInvalid();
+ return VPHeaderPHIRecipe::computeCost(VF, Ctx);
+}
+
void VPWidenPHIRecipe::execute(VPTransformState &State) {
executePhiRecipe(this, *this, State, /*IsScalar=*/false, Name);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index eb5958c37043d..5a2c814069217 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4358,6 +4358,55 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
}
}
+void VPlanTransforms::adjustMonotonicPhiBackedgeUsers(
+ VPlan &Plan, VPBasicBlock *HeaderVPBB, PredicatedScalarEvolution &PSE) {
+ for (VPRecipeBase &R : HeaderVPBB->phis()) {
+ auto *MonotonicPhi = dyn_cast<VPMonotonicPHIRecipe>(&R);
+ if (!MonotonicPhi)
+ continue;
+
+ // Obtain mask value for the predicate edge from the last VPBlendRecipe in
+ // chain.
+ VPValue *Chain = MonotonicPhi->getBackedgeValue();
+ VPValue *Mask = nullptr;
+ while (auto *BlendR = dyn_cast<VPBlendRecipe>(Chain))
+ for (unsigned I = 0, E = BlendR->getNumIncomingValues(); I != E; ++I)
+ if (auto *IncomingVal = BlendR->getIncomingValue(I);
+ IncomingVal != MonotonicPhi) {
+ Chain = IncomingVal;
+ Mask = BlendR->getMask(I);
+ break;
+ }
+ assert(Mask);
+
+ auto &Desc = MonotonicPhi->getDescriptor();
+ auto &SE = *PSE.getSE();
+ auto *Step = vputils::getOrCreateVPValueForSCEVExpr(
+ Plan, Desc.getExpr()->getStepRecurrence(SE));
+
+ auto *BackedgeVal = MonotonicPhi->getIncomingValue(1);
+ auto *InsertBlock = BackedgeVal->getDefiningRecipe()->getParent();
+ VPBuilder Builder(InsertBlock, InsertBlock->getFirstNonPhi());
+
+ Type *UpdateType = MonotonicPhi->getScalarType();
+ if (UpdateType->isPointerTy())
+ UpdateType = Plan.getDataLayout().getIndexType(UpdateType);
+
+ auto *HandledLanes = Builder.createNaryOp(
+ VPInstruction::NumActiveLanes, {Mask}, nullptr, {}, {},
+ DebugLoc::getUnknown(), "handled.lanes", UpdateType);
+ VPValue *Offset =
+ Builder.createOverflowingOp(Instruction::Mul, {Step, HandledLanes});
+ VPValue *Update;
+ if (MonotonicPhi->getScalarType()->isPointerTy())
+ Update = Builder.createPtrAdd(MonotonicPhi, Offset);
+ else
+ Update = Builder.createAdd(MonotonicPhi, Offset, {}, "monotonic.add");
+
+ BackedgeVal->replaceAllUsesWith(Update);
+ }
+}
+
/// Check if \p V is a binary expression of a widened IV and a loop-invariant
/// value. Returns the widened IV if found, nullptr otherwise.
static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1c9959665ee7b..f43becfc60f96 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -155,6 +155,7 @@ struct VPlanTransforms {
const VPDominatorTree &VPDT,
const MapVector<PHINode *, InductionDescriptor> &Inductions,
const MapVector<PHINode *, RecurrenceDescriptor> &Reductions,
+ const MapVector<PHINode *, MonotonicDescriptor> &MonotonicPHIs,
const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering);
@@ -567,6 +568,13 @@ struct VPlanTransforms {
static void adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
VFRange &Range);
+ /// Adjust the backedge value for monotonic PHIs. Changes the update to be
+ /// number of active lanes of the predicated edge between the step increment
+ /// and the loop (multiplied by the step size).
+ static void adjustMonotonicPhiBackedgeUsers(VPlan &Plan,
+ VPBasicBlock *HeaderVPBB,
+ PredicatedScalarEvolution &PSE);
+
/// Optimize FindLast reductions selecting IVs (or expressions of IVs) by
/// converting them to FindIV reductions, if their IV range excludes a
/// suitable sentinel value. For expressions of IVs, the expression is sunk
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index fe44ae5c1f846..d2af7f1499b11 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -445,8 +445,8 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
all_of(VPI->operands(), isSingleScalar));
if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
+ if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe,
+ VPMonotonicPHIRecipe>(VPV))
return true;
if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
return Expr->isVectorToScalar();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll
new file mode 100644
index 0000000000000..d887180de6552
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/compress-idioms.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt < %s -lv-monotonic-patterns=true -mtriple=aarch64 -mattr=+sve2p2 -passes=loop-vectorize -S 2>&1 | FileCheck %s
+
+; SVE compresstore/expandload vectorization (requires +sve2p2 for expandload and +sve for compresstore).
+
+define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define void @compress_store(
+; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext <vscale x 4 x i1> [[TMP4]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
+; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[IF_THEN:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define void @expand_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP11]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.masked.expandload.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP8:%.*]] = zext <vscale x 4 x i1> [[TMP4]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP8]])
+; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP9]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[IF_THEN:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
+ %load.dst = load i32, ptr %dst.ptr, align 4
+ %cmp = icmp slt i32 %load.dst, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %src.idx = sext i32 %idx to i64
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx
+ %load.src = load i32, ptr %src.ptr, align 4
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll
new file mode 100644
index 0000000000000..470e1bbf85ae3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/compress-idioms.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 6
+; RUN: opt -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -mtriple=aarch64 -mattr=+sve2p2 -passes=loop-vectorize -vplan-print-after=printOptimizedVPlan -disable-output %s -S 2>&1 | FileCheck %s
+
+define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'compress_store'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: MONOTONIC-PHI ir<%idx> = phi ir<0>, vp<%monotonic.add>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.src> = load vp<[[VP5]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp slt ir<%load.src>, ir<%c>
+; CHECK-NEXT: EMIT-SCALAR ir<%dst.idx> = sext ir<%idx> to i64
+; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr inbounds ir<%dst>, ir<%dst.idx>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.compressstore(ir<%load.src>, vp<[[VP6]]>, ir<%cmp>)
+; CHECK-NEXT: EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp>
+; CHECK-NEXT: EMIT vp<%monotonic.add> = add ir<%idx>, vp<%handled.lanes>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<%monotonic.add>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'expand_load'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: MONOTONIC-PHI ir<%idx> = phi ir<0>, vp<%monotonic.add>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.dst> = load vp<[[VP5]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp slt ir<%load.dst>, ir<%c>
+; CHECK-NEXT: EMIT-SCALAR ir<%src.idx> = sext ir<%idx> to i64
+; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, ir<%src.idx>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.expandload(vp<[[VP6]]>, ir<%cmp>, ir<poison>)
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, vp<[[VP7]]>, ir<%cmp>
+; CHECK-NEXT: EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp>
+; CHECK-NEXT: EMIT vp<%monotonic.add> = add ir<%idx>, vp<%handled.lanes>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<%monotonic.add>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
+ %load.dst = load i32, ptr %dst.ptr, align 4
+ %cmp = icmp slt i32 %load.dst, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %src.idx = sext i32 %idx to i64
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx
+ %load.src = load i32, ptr %src.ptr, align 4
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a7271524f0191..abdf0754ea4af 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -26,6 +26,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeCallWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustMonotonicPhiBackedgeUsers
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeInductionLiveOutUsers
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
new file mode 100644
index 0000000000000..da7a1f913f6cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
@@ -0,0 +1,424 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^for.body:" --version 5
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-IC1
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-TF
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -force-vector-interleave=2 -passes=loop-vectorize -disable-output -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s --check-prefix=IC2
+
+; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported
+
+define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-IC1-LABEL: define void @test_compress_store_with_index(
+; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-IC1-NEXT: [[ENTRY:.*]]:
+; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC1: [[VECTOR_PH]]:
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC1: [[VECTOR_BODY]]:
+; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]])
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]]
+; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-IC1: [[MIDDLE_BLOCK]]:
+; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC1: [[SCALAR_PH]]:
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
+; CHECK-IC1: [[FOR_INC]]:
+;
+; CHECK-TF-LABEL: define void @test_compress_store_with_index(
+; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]])
+; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[EXIT:.*]]
+; CHECK-TF: [[EXIT]]:
+; CHECK-TF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported
+
+define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-IC1-LABEL: define void @test_expand_load_with_index(
+; CHECK-IC1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-IC1-NEXT: [[ENTRY:.*]]:
+; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC1: [[VECTOR_PH]]:
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC1: [[VECTOR_BODY]]:
+; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-IC1-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]])
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP6]]
+; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-IC1: [[MIDDLE_BLOCK]]:
+; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC1: [[SCALAR_PH]]:
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
+; CHECK-IC1: [[FOR_INC]]:
+;
+; CHECK-TF-LABEL: define void @test_expand_load_with_index(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP4]]
+; CHECK-TF-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[TMP3]], <4 x i32> poison)
+; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
+; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[EXIT:.*]]
+; CHECK-TF: [[EXIT]]:
+; CHECK-TF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
+ %load.dst = load i32, ptr %dst.ptr, align 4
+ %cmp = icmp slt i32 %load.dst, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %src.idx = sext i32 %idx to i64
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx
+ %load.src = load i32, ptr %src.ptr, align 4
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported
+
+define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-IC1-LABEL: define i32 @test_conditionally_incremented_phi_liveout(
+; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-IC1-NEXT: [[ENTRY:.*]]:
+; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC1: [[VECTOR_PH]]:
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC1: [[VECTOR_BODY]]:
+; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]])
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]]
+; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-IC1: [[MIDDLE_BLOCK]]:
+; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC1: [[SCALAR_PH]]:
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
+; CHECK-IC1: [[FOR_INC]]:
+;
+; CHECK-TF-LABEL: define i32 @test_conditionally_incremented_phi_liveout(
+; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]])
+; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[EXIT:.*]]
+; CHECK-TF: [[EXIT]]:
+; CHECK-TF-NEXT: ret i32 [[MONOTONIC_ADD]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %idx.1
+}
+
+; Negative test: Conditional pointer (rather than index) increments are not supported yet (needs LAA support).
+define void @test_compress_store_with_pointer(ptr writeonly noalias %init.dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define void @test_compress_store_with_pointer(
+; CHECK-SAME: ptr noalias writeonly [[INIT_DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[IF_THEN:.*:]]
+; CHECK-NEXT: br label %[[FOR_INC:.*]]
+; CHECK: [[FOR_INC]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %dst = phi ptr [ %init.dst, %entry ], [ %dst.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.inc = getelementptr inbounds i8, ptr %dst, i64 4
+ store i32 %load.src, ptr %dst, align 4
+ br label %for.inc
+
+for.inc:
+ %dst.1 = phi ptr [ %dst.inc, %if.then ], [ %dst, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Negative test: Storing the conditionally incremented phi is invalid (as all uses must be uniform).
+define void @test_store_conditionally_incremented_value(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define void @test_store_conditionally_incremented_value(
+; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[IF_THEN:.*:]]
+; CHECK-NEXT: br label %[[FOR_INC:.*]]
+; CHECK: [[FOR_INC]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %idx, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Pre-increment is currently not matched as we require one use of the step instruction.
+define i32 @test_pre_increment_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define i32 @test_pre_increment_compress_store(
+; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[IF_THEN:.*:]]
+; CHECK-NEXT: br label %[[FOR_INC:.*]]
+; CHECK: [[FOR_INC]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %idx.next = add nsw i32 %idx, 1
+ %dst.idx = sext i32 %idx.next to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %idx.1
+}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3bcbd3ca6c937..6dd891c5e5567 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -93,6 +93,7 @@ class VPlanTestIRBase : public testing::Test {
VPlanTransforms::createHeaderPhiRecipes(
*Plan, PSE, *L, VPDT, Inductions,
MapVector<PHINode *, RecurrenceDescriptor>(),
+ MapVector<PHINode *, MonotonicDescriptor>(),
SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
/*AllowReordering=*/false);
}
>From 74f1bc4cd29dc8f5503b7809e9d563ae224a456a Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 6 Aug 2026 14:50:46 +0000
Subject: [PATCH 2/7] Fix epilogue resume handling
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 10 +-
.../compress-store-vec-epilogue.ll | 98 +++++++++++++++++++
3 files changed, 105 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fb74f9a7bca26..baa7d6f832c87 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7793,7 +7793,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
}
} else {
// Retrieve the induction resume value via ResumeForEpilogue.
- PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
+ assert(isa<VPWidenInductionRecipe>(&R) || isa<VPMonotonicPHIRecipe>(&R));
+ PHINode *IndPhi = cast<VPHeaderPHIRecipe>(&R)->getPHINode();
ResumeV = IRPhiToResumeForEpi.at(IndPhi)->getUnderlyingValue();
}
assert(ResumeV && "Must have a resume value");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bc0d924b91e3f..b738ec4ca1cce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2510,6 +2510,11 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
VPUser::addOperand(V);
}
+ /// Returns the underlying PHINode if one exists, or null otherwise.
+ PHINode *getPHINode() const {
+ return cast_if_present<PHINode>(getUnderlyingValue());
+ }
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -2586,11 +2591,6 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
/// incoming value, its start value.
unsigned getNumIncoming() const override { return 1; }
- /// Returns the underlying PHINode if one exists, or null otherwise.
- PHINode *getPHINode() const {
- return cast_if_present<PHINode>(getUnderlyingValue());
- }
-
/// Returns the induction descriptor for the recipe.
const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
diff --git a/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
new file mode 100644
index 0000000000000..37c605c2b3b00
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^for.body:" --version 5
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=16 -epilogue-vectorization-force-VF=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK
+
+define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-LABEL: define void @compress_store(
+; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 42, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.compressstore.v16i32.p0(<16 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <16 x i1> [[TMP1]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP1]] to <16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[MONOTONIC_IV7:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[MONOTONIC_ADD9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX6]]
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD8]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[MONOTONIC_IV7]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP9]]
+; CHECK-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD8]], ptr align 4 [[TMP10]], <4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP11:%.*]] = zext <4 x i1> [[TMP8]] to <4 x i32>
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]])
+; CHECK-NEXT: [[MONOTONIC_ADD9]] = add i32 [[MONOTONIC_IV7]], [[TMP12]]
+; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX6]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N11]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX12:%.*]] = phi i32 [ [[MONOTONIC_ADD9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 42, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
>From 1cc10fe7e584c765c0c5084ec2d7a7a29fcc13f0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 6 Aug 2026 15:44:58 +0000
Subject: [PATCH 3/7] Don't allow null phi
---
llvm/lib/Transforms/Vectorize/VPlan.h | 10 ++++------
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 2 +-
2 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b738ec4ca1cce..c721d5bcd0313 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2964,9 +2964,9 @@ class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe {
MonotonicDescriptor Desc;
public:
- VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc,
+ VPMonotonicPHIRecipe(PHINode &Phi, const MonotonicDescriptor &Desc,
VPValue &Start, VPValue &BackedgeValue)
- : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, Phi, &Start),
+ : VPHeaderPHIRecipe(VPRecipeBase::VPMonotonicPHISC, &Phi, &Start),
Desc(Desc) {
addOperand(&BackedgeValue);
@@ -2975,10 +2975,8 @@ class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe {
~VPMonotonicPHIRecipe() override = default;
VPMonotonicPHIRecipe *clone() override {
- auto *R =
- new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()), Desc,
- *getStartValue(), *getBackedgeValue());
- return R;
+ return new VPMonotonicPHIRecipe(*getPHINode(), Desc, *getStartValue(),
+ *getBackedgeValue());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPMonotonicPHISC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index aea63707ad5a2..d7d135d761252 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -962,7 +962,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
auto MonotonicIt = MonotonicPHIs.find(Phi);
if (MonotonicIt != MonotonicPHIs.end())
- return new VPMonotonicPHIRecipe(Phi, MonotonicIt->second, *Start,
+ return new VPMonotonicPHIRecipe(*Phi, MonotonicIt->second, *Start,
*BackedgeValue);
assert(Reductions.contains(Phi) && "only reductions are expected now");
>From 58158c356b3405dcb6d1f20472d3ac99b1c0c926 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 6 Aug 2026 19:30:51 +0000
Subject: [PATCH 4/7] Move negative tests to new file
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 -
.../compress-idioms-negative-tests.ll | 94 ++++++++++++++++
.../LoopVectorize/compress-idioms.ll | 105 ------------------
3 files changed, 94 insertions(+), 106 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/compress-idioms-negative-tests.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 53b58af954782..6358d74f02f96 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1231,7 +1231,6 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked
: TTI::CastContextHint::Normal;
}
-
const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
if (WidenMemoryRecipe == nullptr)
return TTI::CastContextHint::None;
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms-negative-tests.ll
new file mode 100644
index 0000000000000..6071fe4800b27
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms-negative-tests.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -disable-output -pass-remarks-analysis=".*" 2>&1 | FileCheck %s
+
+; CHECK: loop not vectorized: cannot identify array bounds
+
+; Negative test: Conditional pointer (rather than index) increments are not supported yet (needs LAA support).
+define void @test_compress_store_with_pointer(ptr writeonly noalias %init.dst, ptr readonly %src, i32 %c, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %dst = phi ptr [ %init.dst, %entry ], [ %dst.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.inc = getelementptr inbounds i8, ptr %dst, i64 4
+ store i32 %load.src, ptr %dst, align 4
+ br label %for.inc
+
+for.inc:
+ %dst.1 = phi ptr [ %dst.inc, %if.then ], [ %dst, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK: Recipe with invalid costs prevented vectorization at VF=(2, 4): phi
+
+; Negative test: Storing the conditionally incremented phi is invalid (as all uses must be uniform).
+define void @test_store_conditionally_incremented_value(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %idx, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK: loop not vectorized: value that could not be identified as reduction is used outside the loop
+
+; Pre-increment is currently not matched as we require one use of the step instruction.
+define i32 @test_pre_increment_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br i1 %cmp, label %if.then, label %for.inc
+
+if.then:
+ %idx.next = add nsw i32 %idx, 1
+ %dst.idx = sext i32 %idx.next to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %idx.1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
index da7a1f913f6cd..eef74fb277532 100644
--- a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
@@ -317,108 +317,3 @@ for.inc:
exit:
ret i32 %idx.1
}
-
-; Negative test: Conditional pointer (rather than index) increments are not supported yet (needs LAA support).
-define void @test_compress_store_with_pointer(ptr writeonly noalias %init.dst, ptr readonly %src, i32 %c, i64 %n) {
-; CHECK-LABEL: define void @test_compress_store_with_pointer(
-; CHECK-SAME: ptr noalias writeonly [[INIT_DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[IF_THEN:.*:]]
-; CHECK-NEXT: br label %[[FOR_INC:.*]]
-; CHECK: [[FOR_INC]]:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %dst = phi ptr [ %init.dst, %entry ], [ %dst.1, %for.inc ]
- %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
- %load.src = load i32, ptr %src.ptr, align 4
- %cmp = icmp slt i32 %load.src, %c
- br i1 %cmp, label %if.then, label %for.inc
-
-if.then:
- %dst.inc = getelementptr inbounds i8, ptr %dst, i64 4
- store i32 %load.src, ptr %dst, align 4
- br label %for.inc
-
-for.inc:
- %dst.1 = phi ptr [ %dst.inc, %if.then ], [ %dst, %for.body ]
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-; Negative test: Storing the conditionally incremented phi is invalid (as all uses must be uniform).
-define void @test_store_conditionally_incremented_value(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
-; CHECK-LABEL: define void @test_store_conditionally_incremented_value(
-; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[IF_THEN:.*:]]
-; CHECK-NEXT: br label %[[FOR_INC:.*]]
-; CHECK: [[FOR_INC]]:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
- %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
- %load.src = load i32, ptr %src.ptr, align 4
- %cmp = icmp slt i32 %load.src, %c
- br i1 %cmp, label %if.then, label %for.inc
-
-if.then:
- %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
- store i32 %idx, ptr %dst.ptr, align 4
- %idx.next = add nsw i32 %idx, 1
- br label %for.inc
-
-for.inc:
- %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-; Pre-increment is currently not matched as we require one use of the step instruction.
-define i32 @test_pre_increment_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
-; CHECK-LABEL: define i32 @test_pre_increment_compress_store(
-; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[IF_THEN:.*:]]
-; CHECK-NEXT: br label %[[FOR_INC:.*]]
-; CHECK: [[FOR_INC]]:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
- %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
- %load.src = load i32, ptr %src.ptr, align 4
- %cmp = icmp slt i32 %load.src, %c
- br i1 %cmp, label %if.then, label %for.inc
-
-if.then:
- %idx.next = add nsw i32 %idx, 1
- %dst.idx = sext i32 %idx.next to i64
- %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
- store i32 %load.src, ptr %dst.ptr, align 4
- br label %for.inc
-
-for.inc:
- %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %exit, label %for.body
-
-exit:
- ret i32 %idx.1
-}
>From 818f095f1e5dfeba9fc466b757290a5e47d72e79 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 7 Aug 2026 10:11:46 +0000
Subject: [PATCH 5/7] Test uncond increment
---
.../LoopVectorize/compress-idioms.ll | 98 ++++++++++++++++++-
1 file changed, 96 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
index eef74fb277532..ca2824612fa94 100644
--- a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^for.body:" --version 5
-; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-IC1
-; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-TF
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefix=CHECK-IC1
+; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S 2>&1 | FileCheck %s -check-prefix=CHECK-TF
; RUN: opt < %s -lv-monotonic-patterns=true -force-target-supports-masked-memory-ops -force-vector-width=4 -force-vector-interleave=2 -passes=loop-vectorize -disable-output -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s --check-prefix=IC2
; IC2: loop not vectorized: Interleaving of loops with monotonic vars is not supported
@@ -317,3 +317,97 @@ for.inc:
exit:
ret i32 %idx.1
}
+
+; An unconditional increment should lower as a simple induction (not a monotonic PHI).
+define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
+; CHECK-IC1-LABEL: define void @test_unconditional_increment(
+; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-IC1-NEXT: [[ENTRY:.*]]:
+; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-IC1: [[VECTOR_PH]]:
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = add i32 15, [[TMP0]]
+; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-IC1: [[VECTOR_BODY]]:
+; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-IC1-NEXT: [[TMP7:%.*]] = add i32 15, [[TMP1]]
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = sext i32 [[TMP7]] to i64
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
+; CHECK-IC1-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP4]], align 4
+; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-IC1: [[MIDDLE_BLOCK]]:
+; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-IC1: [[SCALAR_PH]]:
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 15, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-IC1: [[FOR_BODY]]:
+;
+; CHECK-TF-LABEL: define void @test_unconditional_increment(
+; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-TF-NEXT: [[TMP6:%.*]] = add i32 15, [[TMP1]]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = sext i32 [[TMP6]] to i64
+; CHECK-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP4]], <4 x i1> [[TMP0]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-TF-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[EXIT:.*]]
+; CHECK-TF: [[EXIT]]:
+; CHECK-TF-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %idx = phi i32 [ 15, %entry ], [ %idx.1, %for.inc ]
+ %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load.src = load i32, ptr %src.ptr, align 4
+ %cmp = icmp slt i32 %load.src, %c
+ br label %inc.step
+
+inc.step:
+ %dst.idx = sext i32 %idx to i64
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ store i32 %load.src, ptr %dst.ptr, align 4
+ %idx.next = add nsw i32 %idx, 1
+ br label %for.inc
+
+for.inc:
+ %idx.1 = phi i32 [ %idx.next, %inc.step ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
>From c2b6f45b4c20d30b15dc5741f777761a5776d713 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 7 Aug 2026 10:48:46 +0000
Subject: [PATCH 6/7] Rebase fixups
---
.../Transforms/LoopVectorize/compress-idioms.ll | 16 ++++++++--------
.../LoopVectorize/compress-store-vec-epilogue.ll | 6 +++---
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
index ca2824612fa94..57fe9a95d1d2d 100644
--- a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
@@ -12,7 +12,7 @@ define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr read
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -47,7 +47,7 @@ define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr read
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -116,7 +116,7 @@ define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i3
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -152,7 +152,7 @@ define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i3
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -223,7 +223,7 @@ define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %ds
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -258,7 +258,7 @@ define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %ds
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -326,7 +326,7 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-IC1-NEXT: [[TMP0:%.*]] = trunc i64 [[N_VEC]] to i32
; CHECK-IC1-NEXT: [[TMP6:%.*]] = add i32 15, [[TMP0]]
@@ -358,7 +358,7 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
index 37c605c2b3b00..ead829cdeee5b 100644
--- a/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/compress-store-vec-epilogue.ll
@@ -11,7 +11,7 @@ define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
@@ -40,7 +40,7 @@ define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -65,7 +65,7 @@ define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %
; CHECK-NEXT: br i1 [[CMP_N11]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX12:%.*]] = phi i32 [ [[MONOTONIC_ADD9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i32 [ [[MONOTONIC_ADD9]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[MONOTONIC_ADD]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 42, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
;
>From afd9dde1c183166e7553efd4a28905a91bf485ea Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 10 Aug 2026 12:36:12 +0000
Subject: [PATCH 7/7] Fixups
---
.../Vectorize/LoopVectorizationLegality.h | 16 +-
.../Vectorize/LoopVectorizationLegality.cpp | 26 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 79 +++---
.../Transforms/Vectorize/VPRecipeBuilder.h | 5 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +
.../LoopVectorize/compress-idioms.ll | 230 +++++++++---------
6 files changed, 199 insertions(+), 161 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index f3ba591429851..743634a5cfb3b 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -392,9 +392,6 @@ class LoopVectorizationLegality {
/// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
LLVM_ABI int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
- /// Check if memory access is compressed when vectorizing.
- bool isCompressedPtr(Type *AccessTy, Value *Ptr, BasicBlock *BB) const;
-
/// Returns true if \p V is invariant across all loop iterations according to
/// SCEV.
LLVM_ABI bool isInvariant(Value *V) const;
@@ -487,6 +484,12 @@ class LoopVectorizationLegality {
/// Returns a list of all known histogram operations in the loop.
bool hasHistograms() const { return !Histograms.empty(); }
+ /// Returns true if \p I is a compressed load or store (which can map to a
+ /// llvm.masked.expandload or llvm.masked.compressstore).
+ bool isCompressedLoadOrStore(const Instruction *I) const {
+ return CompressedMemoryOps.contains(I);
+ }
+
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
return &PSE;
}
@@ -657,6 +660,9 @@ class LoopVectorizationLegality {
/// better choice for the main induction than the existing one.
void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID);
+ /// Check if memory access is compressed when vectorizing.
+ bool isCompressedPtr(Type *AccessTy, Value *Ptr, BasicBlock *AccessBB) const;
+
/// The loop that we evaluate.
Loop *TheLoop;
@@ -741,6 +747,10 @@ class LoopVectorizationLegality {
/// may work on the same memory location.
SmallVector<HistogramInfo, 1> Histograms;
+ /// Contains all identified compressed loads/stores. This are loads/stores to
+ /// a 'compressed' pointer as defined by isCompressedPtr.
+ SmallPtrSet<const Instruction *, 8> CompressedMemoryOps;
+
/// Whether or not creating SCEV predicates is allowed.
bool AllowRuntimeSCEVChecks;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 7189d345f6d6c..203266e33178d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -478,10 +478,7 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
}
bool LoopVectorizationLegality::isCompressedPtr(Type *AccessTy, Value *Ptr,
- BasicBlock *BB) const {
- if (!EnableMonotonicPatterns)
- return false;
-
+ BasicBlock *AccessBB) const {
MonotonicDescriptor Desc;
if (!MonotonicDescriptor::isMonotonicVal(Ptr, TheLoop, Desc, *PSE.getSE()))
return false;
@@ -489,15 +486,13 @@ bool LoopVectorizationLegality::isCompressedPtr(Type *AccessTy, Value *Ptr,
// Check that the memory operation has the same predicate as the step.
// TODO: Relax these restrictions.
if (Desc.getPredicateEdge() !=
- MonotonicDescriptor::Edge(BB, BB->getUniqueSuccessor()))
+ MonotonicDescriptor::Edge(AccessBB, AccessBB->getUniqueSuccessor()))
return false;
// Check if pointer step equals access size.
auto *Step =
dyn_cast<SCEVConstant>(Desc.getExpr()->getStepRecurrence(*PSE.getSE()));
- if (!Step)
- return false;
- return Step->getAPInt() == BB->getDataLayout().getTypeAllocSize(AccessTy);
+ return Step == PSE.getSE()->getSizeOfExpr(Step->getType(), AccessTy);
}
bool LoopVectorizationLegality::isInvariant(Value *V) const {
@@ -1091,6 +1086,21 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
Hints->setPotentiallyUnsafe();
}
+ if (isa<LoadInst, StoreInst>(I)) {
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ Type *AccessTy = getLoadStoreType(&I);
+ if (ConditionallyExecutedOps.contains(&I) &&
+ isCompressedPtr(AccessTy, Ptr, I.getParent())) {
+ if (!EnableMonotonicPatterns) {
+ reportVectorizationFailure(
+ "compressed load/store vectorization not enabled",
+ "CantVectorizeCompressedLoadStore", ORE, TheLoop, &I);
+ return false;
+ }
+ CompressedMemoryOps.insert(&I);
+ }
+ }
+
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index baa7d6f832c87..82f976b30193c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -873,7 +873,6 @@ class LoopVectorizationCostModel {
CM_Widen_Reverse, // For consecutive accesses with stride -1.
CM_Interleave,
CM_GatherScatter,
- CM_Compressed,
CM_Scalarize,
/// A widening decision that has been invalidated after replacing the
/// corresponding recipe during VPlan transforms.
@@ -2421,8 +2420,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
Type *ScalarTy = getLoadStoreType(I);
Value *Ptr = getLoadStorePointerOperand(I);
bool IsConsecutive = Legal->isConsecutivePtr(ScalarTy, Ptr);
- bool IsCompressed =
- !IsConsecutive && Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
+ bool IsCompressed = !IsConsecutive && Legal->isCompressedLoadOrStore(I);
return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
!(IsCompressed && isLegalExpandLoadOrCompressStore(I)) &&
!Config.isLegalGatherOrScatter(I, VF);
@@ -2664,8 +2662,10 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
auto *ScalarTy = getLoadStoreType(I);
// In order to be widened, the pointer should be consecutive or compressed.
- bool Compressed = Legal->isCompressedPtr(ScalarTy, Ptr, I->getParent());
- int Stride = Compressed ? 1 : Legal->isConsecutivePtr(ScalarTy, Ptr);
+ int Stride = Legal->isConsecutivePtr(ScalarTy, Ptr);
+ if (!Stride && Legal->isCompressedLoadOrStore(I))
+ Stride = 1;
+
if (!Stride)
return std::nullopt;
@@ -2680,8 +2680,6 @@ LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
if (hasIrregularType(ScalarTy, DL))
return std::nullopt;
- if (Compressed)
- return CM_Compressed;
return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
}
@@ -2775,9 +2773,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (IsUniformMemOpUse(I))
return true;
- return (
- WideningDecision == CM_Widen || WideningDecision == CM_Widen_Reverse ||
- WideningDecision == CM_Interleave || WideningDecision == CM_Compressed);
+ return (WideningDecision == CM_Widen ||
+ WideningDecision == CM_Widen_Reverse ||
+ WideningDecision == CM_Interleave);
};
// Returns true if Ptr is the pointer operand of a memory access instruction
@@ -4079,7 +4077,7 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
// 4. Compressed loads/stores (which do not support scalarization)
if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
!useEmulatedMaskMemRefHack(&I, VF) &&
- getWideningDecision(&I, VF) != CM_Compressed &&
+ !Legal->isCompressedLoadOrStore(&I) &&
computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
for (const auto &[I, IC] : ScalarCosts)
ScalarCostsVF.insert({I, IC});
@@ -4329,9 +4327,8 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *I, ElementCount VF, InstWidening Kind) {
- assert(
- (Kind == CM_Widen || Kind == CM_Widen_Reverse || Kind == CM_Compressed) &&
- "Expected a consecutive widening decision");
+ assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
+ "Expected a consecutive widening decision");
Type *ValTy = getLoadStoreType(I);
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(I);
@@ -4341,7 +4338,7 @@ InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
if (isMaskRequired(I)) {
Intrinsic::ID LoadIID = Intrinsic::masked_load;
Intrinsic::ID StoreIID = Intrinsic::masked_store;
- if (Kind == CM_Compressed) {
+ if (Legal->isCompressedLoadOrStore(I)) {
LoadIID = Intrinsic::masked_expandload;
StoreIID = Intrinsic::masked_compressstore;
}
@@ -5318,14 +5315,15 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI::CastContextHint::Interleave;
case LoopVectorizationCostModel::CM_Scalarize:
case LoopVectorizationCostModel::CM_Widen:
+ // TODO: Add 'Compressed' hint (not needed for any targets yet).
+ if (Legal->isCompressedLoadOrStore(I))
+ return TTI::CastContextHint::None;
return isPredicatedInst(I) ? TTI::CastContextHint::Masked
: TTI::CastContextHint::Normal;
case LoopVectorizationCostModel::CM_Widen_Reverse:
return TTI::CastContextHint::Reversed;
case LoopVectorizationCostModel::CM_Unknown:
llvm_unreachable("Instr did not go through cost modelling?");
- case LoopVectorizationCostModel::CM_Compressed:
- // TODO: Add Compressed hint (not needed for any targets yet).
case LoopVectorizationCostModel::CM_InvalidatedDecision:
return TTI::CastContextHint::None;
}
@@ -6286,10 +6284,10 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
// reverse consecutive.
LoopVectorizationCostModel::InstWidening Decision =
CM.getWideningDecision(I, Range.Start);
+
bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
- bool Compressed = Decision == LoopVectorizationCostModel::CM_Compressed;
bool Consecutive =
- Reverse || Compressed || Decision == LoopVectorizationCostModel::CM_Widen;
+ Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
@@ -6304,13 +6302,6 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
- Type *LoadTy = Load->getType();
-
- if (Compressed)
- return Builder.createWidenMemIntrinsic(
- Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(LoadTy)},
- LoadTy, Load->getAlign(), *VPI, Load->getDebugLoc());
-
auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI,
Load->getDebugLoc());
if (Reverse)
@@ -6321,12 +6312,6 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
StoreInst *Store = cast<StoreInst>(I);
VPValue *StoredVal = VPI->getOperand(0);
- if (Compressed)
- return Builder.createWidenMemIntrinsic(
- Intrinsic::masked_compressstore, {StoredVal, Ptr, Mask},
- StoredVal->getScalarType(), Store->getAlign(), *VPI,
- Store->getDebugLoc());
-
if (Reverse)
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
Store->getDebugLoc());
@@ -6457,6 +6442,36 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
VPI->getDebugLoc());
}
+VPWidenMemIntrinsicRecipe *
+VPRecipeBuilder::widenIfCompressedMemoryOp(VPInstruction *VPI) {
+ Instruction *I = VPI->getUnderlyingInstr();
+ if (!Legal->isCompressedLoadOrStore(I))
+ return nullptr;
+
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(VPI);
+
+ VPValue *Mask = VPI->getMask();
+ Type *AccessTy = getLoadStoreType(I);
+ Align Alignment = getLoadStoreAlignment(I);
+
+ VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
+ Ptr = Builder.createConsecutiveVectorPointer(Ptr, AccessTy,
+ /*Reverse=*/false,
+ VPI->getDebugLoc());
+
+ if (VPI->getOpcode() == Instruction::Load)
+ return new VPWidenMemIntrinsicRecipe(
+ Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(AccessTy)},
+ AccessTy, Alignment, *VPI, I->getDebugLoc());
+
+ VPValue *StoredValue = VPI->getOperand(0);
+ return new VPWidenMemIntrinsicRecipe(Intrinsic::masked_compressstore,
+ {StoredValue, Ptr, Mask}, AccessTy,
+ Alignment, *VPI, I->getDebugLoc());
+}
+
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
StoreInst *SI;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 1303b62d5faf4..6adf3961bcbb4 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -78,6 +78,11 @@ class VPRecipeBuilder {
/// scalar loop.
VPHistogramRecipe *widenIfHistogram(VPInstruction *VPI);
+ /// If \p VPI represents a compressed memory operation (as determined by
+ /// LoopVectorizationLegality) widen it by emitting an llvm.masked.expandload
+ /// or llvm.masked.compresstore (for loads and stores respectively).
+ VPWidenMemIntrinsicRecipe *widenIfCompressedMemoryOp(VPInstruction *VPI);
+
/// If \p VPI is a store of a reduction into an invariant address, delete it.
/// If it is the final store of a reduction result, a uniform store recipe
/// will be created for it in the middle block. Returns `true` if replacement
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5a2c814069217..3c4e344cabd9e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5423,6 +5423,10 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
return ReplaceWith(VPI, VPBuilder(VPI).insert(Histogram));
+ if (VPWidenMemIntrinsicRecipe *Compressed =
+ RecipeBuilder.widenIfCompressedMemoryOp(VPI))
+ return ReplaceWith(VPI, VPBuilder(VPI).insert(Compressed));
+
return false;
});
diff --git a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
index 57fe9a95d1d2d..068be3ac4a77b 100644
--- a/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
+++ b/llvm/test/Transforms/LoopVectorize/compress-idioms.ll
@@ -12,34 +12,34 @@ define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr read
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-IC1: [[VECTOR_BODY]]:
; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
-; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]])
-; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
-; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
-; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]]
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
+; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP4]], <4 x i1> [[TMP2]])
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP6]]
; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-IC1: [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1: [[SCALAR_PH]]:
; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
-; CHECK-IC1: [[FOR_INC]]:
+; CHECK-IC1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-IC1: [[FOR_BODY]]:
;
; CHECK-TF-LABEL: define void @test_compress_store_with_index(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
@@ -47,8 +47,8 @@ define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr read
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -59,21 +59,21 @@ define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr read
; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
-; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
-; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]])
-; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
-; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP6]], <4 x i1> [[TMP4]])
+; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]]
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br label %[[EXIT:.*]]
; CHECK-TF: [[EXIT]]:
@@ -116,35 +116,35 @@ define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i3
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-IC1: [[VECTOR_BODY]]:
; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
-; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-IC1-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; CHECK-IC1-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]])
-; CHECK-IC1-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
-; CHECK-IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
-; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP6]]
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-IC1-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]]
; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-IC1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-IC1: [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1: [[SCALAR_PH]]:
; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
-; CHECK-IC1: [[FOR_INC]]:
+; CHECK-IC1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-IC1: [[FOR_BODY]]:
;
; CHECK-TF-LABEL: define void @test_expand_load_with_index(
; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
@@ -152,8 +152,8 @@ define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i3
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -164,22 +164,22 @@ define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i3
; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
-; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP4]]
-; CHECK-TF-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[TMP3]], <4 x i32> poison)
-; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
-; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
-; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP6]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP7]], ptr align 4 [[TMP2]], <4 x i1> [[TMP4]])
+; CHECK-TF-NEXT: [[TMP8:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP9]]
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br label %[[EXIT:.*]]
; CHECK-TF: [[EXIT]]:
@@ -223,34 +223,34 @@ define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %ds
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-IC1: [[VECTOR_BODY]]:
; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-IC1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-IC1-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-IC1-NEXT: [[TMP2:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
-; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]])
-; CHECK-IC1-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
-; CHECK-IC1-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
-; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP5]]
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
+; CHECK-IC1-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP4]], <4 x i1> [[TMP2]])
+; CHECK-IC1-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-IC1-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP6]]
; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-IC1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-IC1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-IC1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-IC1-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-IC1: [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1: [[SCALAR_PH]]:
; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[MONOTONIC_ADD]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-IC1-NEXT: br label %[[FOR_INC:.*]]
-; CHECK-IC1: [[FOR_INC]]:
+; CHECK-IC1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-IC1: [[FOR_BODY]]:
;
; CHECK-TF-LABEL: define i32 @test_conditionally_incremented_phi_liveout(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
@@ -258,8 +258,8 @@ define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %ds
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -270,21 +270,21 @@ define i32 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %ds
; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[MONOTONIC_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[MONOTONIC_ADD:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
-; CHECK-TF-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-TF-NEXT: [[TMP4:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
-; CHECK-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
-; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP3]])
-; CHECK-TF-NEXT: [[TMP6:%.*]] = zext <4 x i1> [[TMP3]] to <4 x i32>
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
-; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP7]]
+; CHECK-TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sext i32 [[MONOTONIC_IV]] to i64
+; CHECK-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NEXT: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP6]], <4 x i1> [[TMP4]])
+; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-TF-NEXT: [[MONOTONIC_ADD]] = add i32 [[MONOTONIC_IV]], [[TMP8]]
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br label %[[EXIT:.*]]
; CHECK-TF: [[EXIT]]:
@@ -326,19 +326,16 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1: [[VECTOR_PH]]:
-; CHECK-IC1-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-IC1-NEXT: [[TMP0:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-IC1-NEXT: [[TMP6:%.*]] = add i32 15, [[TMP0]]
+; CHECK-IC1-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-IC1-NEXT: [[TMP1:%.*]] = add i64 15, [[N_VEC]]
; CHECK-IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-IC1: [[VECTOR_BODY]]:
; CHECK-IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-IC1-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
-; CHECK-IC1-NEXT: [[TMP7:%.*]] = add i32 15, [[TMP1]]
-; CHECK-IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-IC1-NEXT: [[TMP3:%.*]] = sext i32 [[TMP7]] to i64
-; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
+; CHECK-IC1-NEXT: [[TMP2:%.*]] = add i64 15, [[INDEX]]
+; CHECK-IC1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
; CHECK-IC1-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP4]], align 4
; CHECK-IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -348,7 +345,7 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1: [[SCALAR_PH]]:
; CHECK-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-IC1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 15, %[[ENTRY]] ]
+; CHECK-IC1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 15, %[[ENTRY]] ]
; CHECK-IC1-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-IC1: [[FOR_BODY]]:
;
@@ -358,8 +355,8 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-TF: [[VECTOR_PH]]:
; CHECK-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -367,14 +364,12 @@ define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readon
; CHECK-TF: [[VECTOR_BODY]]:
; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
-; CHECK-TF-NEXT: [[TMP6:%.*]] = add i32 15, [[TMP1]]
-; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP0]], <4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP3:%.*]] = sext i32 [[TMP6]] to i64
-; CHECK-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP4]], <4 x i1> [[TMP0]])
+; CHECK-TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = add i64 15, [[INDEX]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP4]], <4 x i1> [[TMP1]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-TF-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -389,21 +384,20 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %idx = phi i32 [ 15, %entry ], [ %idx.1, %for.inc ]
+ %idx = phi i64 [ 15, %entry ], [ %idx.1, %for.inc ]
%src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
%load.src = load i32, ptr %src.ptr, align 4
%cmp = icmp slt i32 %load.src, %c
br label %inc.step
inc.step:
- %dst.idx = sext i32 %idx to i64
- %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %dst.idx
+ %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
store i32 %load.src, ptr %dst.ptr, align 4
- %idx.next = add nsw i32 %idx, 1
+ %idx.next = add nsw i64 %idx, 1
br label %for.inc
for.inc:
- %idx.1 = phi i32 [ %idx.next, %inc.step ]
+ %idx.1 = phi i64 [ %idx.next, %inc.step ]
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %exit, label %for.body
More information about the llvm-branch-commits
mailing list