[llvm] [LV] Add support for partial alias masking with tail folding (PR #182457)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue May 12 08:17:18 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/182457
>From 7d71765158f59cefd12c18f1cb94360816b548bb Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 19 Feb 2026 15:29:32 +0000
Subject: [PATCH 1/7] [LV] Add support for partial alias masking with tail
folding
This patch adds basic support for partial alias masking, which allows
entering the vector loop even when there is aliasing within a single
vector iteration. It does this by clamping the VF to the safe distance
between pointers. This allows the runtime VF to be anywhere from 2 to
the "static" VF.
Conceptually, this transform looks like:
```
// `c` and `b` may alias.
for (int i = 0; i < n; i++) {
c[i] = a[i] + b[i];
}
```
->
```
svbool_t alias_mask = loop.dependence.war.mask(b, c);
int num_active = num_active_lanes(mask);
if (num_active >= 2) {
for (int i = 0; i < n; i += num_active) {
// ... vector loop masked with `alias_mask`
}
}
// ... scalar tail
```
This initial patch has a number of limitations:
- The loop must be tail-folded
* We intend to follow-up with full alias-masking support for loops
without tail-folding
- The mask and transform is only valid for IC = 1
* Some recipes may not handle the "ClampedVF" correctly at IC > 1
* Note: On AArch64, we also only have native alias mask instructions
for IC = 1
- Reverse iteration is not supported
* The mask reversal logic is not correct for the alias mask
(or clamped ALM)
- First order recurrences are not supported
* The `splice.right` is not lowered correctly for clamped VFs
- This style of vectorization is not enabled by default/costed
* It can be enabled with `-force-partial-aliasing-vectorization`
* When enabled, alias masking is used instead of the standard diff
checks (when legal to do so)
This PR supersedes #100579 (closes #100579).
Rebase tests
Fixups
Fixups
Fixups
Fixups
Fixups
Fixups
Add test
Fixups
Attach alias-mask early
Use VPInstruction in preheader
Fixups
---
llvm/lib/Analysis/VectorUtils.cpp | 2 +
.../Transforms/Vectorize/LoopVectorize.cpp | 100 +++-
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 +
.../Vectorize/VPlanConstruction.cpp | 10 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 26 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 120 ++++-
.../Transforms/Vectorize/VPlanTransforms.h | 17 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 34 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +
.../LoopVectorize/AArch64/alias-mask.ll | 454 ++++++++++++++++++
.../RISCV/alias-mask-force-evl.ll | 64 +++
.../AArch64/vplan-printing-alias-mask.ll | 91 ++++
.../VPlan/vplan-printing-alias-mask.ll | 165 +++++++
.../LoopVectorize/VPlan/vplan-printing.ll | 12 +-
.../alias-mask-data-tail-folding-style.ll | 79 +++
.../alias-mask-negative-tests.ll | 147 ++++++
.../Transforms/LoopVectorize/alias-mask.ll | 382 +++++++++++++++
.../LoopVectorize/pointer-induction.ll | 10 +-
.../remove-redundant-trip-count-scev.ll | 71 +++
.../reuse-lcssa-phi-scev-expansion.ll | 12 +-
21 files changed, 1779 insertions(+), 32 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/alias-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 8b4d6d9652414..635e34af0b6fa 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -167,6 +167,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
return (ScalarOpdIdx == 2);
case Intrinsic::experimental_vp_splice:
return ScalarOpdIdx == 2 || ScalarOpdIdx == 4;
+ case Intrinsic::loop_dependence_war_mask:
+ return true;
default:
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 61c2d3cd228ec..576fa50be565a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -170,6 +170,8 @@ STATISTIC(LoopsVectorized, "Number of loops vectorized");
STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
+STATISTIC(LoopsPartialAliasVectorized,
+ "Number of partial aliasing loops vectorized");
static cl::opt<bool> EnableEpilogueVectorization(
"enable-epilogue-vectorization", cl::init(true), cl::Hidden,
@@ -198,6 +200,10 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
+static cl::opt<bool> ForcePartialAliasingVectorization(
+ "force-partial-aliasing-vectorization", cl::init(false), cl::Hidden,
+ cl::desc("Replace pointer diff checks with alias masks."));
+
/// Option tail-folding-policy indicates that an epilogue is undesired, that
/// tail folding is preferred, and this lists all options. I.e., the vectorizer
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
@@ -1226,6 +1232,50 @@ class LoopVectorizationCostModel {
return getTailFoldingStyle() != TailFoldingStyle::None;
}
+ void tryToEnablePartialAliasMasking() {
+ assert(!IsPartialAliasMaskingEnabled && "Alias masking already enabled!");
+ assert(foldTailByMasking() && "Expected tail folding to be enabled!");
+ assert(!foldTailWithEVL() &&
+ "Did not expect to enable alias masking with EVL!");
+ // Note: FixedOrderRecurrences are not supported yet as we cannot handle
+ // the required `splice.right` with the alias-mask.
+ if (!ForcePartialAliasingVectorization ||
+ !Legal->getFixedOrderRecurrences().empty())
+ return;
+
+ const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
+ if (!Checks)
+ return;
+
+ auto DiffChecks = Checks->getDiffChecks();
+ if (!DiffChecks || DiffChecks->empty())
+ return;
+
+ for (BasicBlock *BB : TheLoop->blocks()) {
+ for (Instruction &I : *BB) {
+ if (!isa<LoadInst, StoreInst>(I)) {
+ assert(isa<CallInst>(I) || !I.mayReadOrWriteMemory());
+ continue;
+ }
+
+ Type *ScalarTy = getLoadStoreType(&I);
+ Value *Ptr = getLoadStorePointerOperand(&I);
+
+ // Currently, we can't handle alias masking in reverse. Reversing the
+ // alias mask is not correct (or necessary). When combined with
+ // tail-folding the active lane mask should only be reversed where the
+ // alias-mask is true.
+ if (Legal->isConsecutivePtr(ScalarTy, Ptr) != 1)
+ return;
+ }
+ }
+
+ IsPartialAliasMaskingEnabled = true;
+ }
+
+ /// Returns true if all loop blocks should have partial aliases masked.
+ bool maskPartialAliasing() const { return IsPartialAliasMaskingEnabled; }
+
/// Returns true if the use of wide lane masks is requested and the loop is
/// using tail-folding with a lane mask for control flow.
bool useWideActiveLaneMask() const {
@@ -1367,6 +1417,9 @@ class LoopVectorizationCostModel {
/// Control finally chosen tail folding style.
TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
+ /// True if partial alias masking is enabled.
+ bool IsPartialAliasMaskingEnabled = false;
+
/// A map holding scalar costs for different vectorization factors. The
/// presence of a cost for an instruction in the mapping indicates that the
/// instruction will be scalarized when vectorizing with the associated
@@ -1555,14 +1608,18 @@ class GeneratedRTChecks {
/// The kind of cost that we are calculating
TTI::TargetCostKind CostKind;
+ /// True if the loop is alias-masked (which allows us to omit diff checks).
+ bool LoopUsesAliasMasking = false;
+
public:
GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
LoopInfo *LI, TargetTransformInfo *TTI,
- TTI::TargetCostKind CostKind)
+ TTI::TargetCostKind CostKind, bool LoopUsesAliasMasking)
: DT(DT), LI(LI), TTI(TTI),
SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
- PSE(PSE), CostKind(CostKind) {}
+ PSE(PSE), CostKind(CostKind),
+ LoopUsesAliasMasking(LoopUsesAliasMasking) {}
/// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
/// accurately estimate the cost of the runtime checks. The blocks are
@@ -1615,7 +1672,7 @@ class GeneratedRTChecks {
}
const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
- if (RtPtrChecking.Need) {
+ if (RtPtrChecking.Need && !LoopUsesAliasMasking) {
auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
"vector.memcheck");
@@ -3048,6 +3105,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
assert(ContainsScalableVF && "Expected scalable vector factor.");
MaxFactors.FixedVF = ElementCount::getFixed(1);
+ } else {
+ tryToEnablePartialAliasMasking();
}
return MaxFactors;
}
@@ -3491,6 +3550,13 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
return nullptr;
}
+ if (CM.maskPartialAliasing()) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LEV: Epilogue vectorization not supported with alias masking.\n");
+ return nullptr;
+ }
+
// Not really a cost consideration, but check for unsupported cases here to
// simplify the logic.
if (!isCandidateForEpilogueVectorization(MainPlan)) {
@@ -6115,6 +6181,14 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan, BestVF, VScale);
}
+ if (CM.maskPartialAliasing()) {
+ assert(CM.foldTailByMasking() && "Expected tail folding to be enabled");
+ VPlanTransforms::materializeAliasMaskCheckBlock(
+ BestVPlan, *CM.Legal->getRuntimePointerChecking()->getDiffChecks(),
+ HasBranchWeights);
+ ++LoopsPartialAliasVectorized;
+ }
+
// Retrieving VectorPH now when it's easier while VPlan still has Regions.
VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
@@ -7073,6 +7147,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::addActiveLaneMask, *Plan, ForControlFlow);
}
+ if (CM.maskPartialAliasing())
+ RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
+
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
return Plan;
}
@@ -8173,7 +8250,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
LVP.emitInvalidCostRemarks(ORE);
- GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind);
+ assert((IsInnerLoop || !CM.maskPartialAliasing()) &&
+ "Did not expect to alias-mask outer loop");
+
+ GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
+ CM.maskPartialAliasing());
if (IsInnerLoop && LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
@@ -8282,6 +8363,17 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
+ if (CM.maskPartialAliasing()) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Not interleaving due to partial aliasing vectorization.\n");
+ IntDiagMsg = {
+ "PartialAliasingVectorization",
+ "Unable to interleave due to partial aliasing vectorization."};
+ InterleaveLoop = false;
+ IC = 1;
+ }
+
// Emit diagnostic messages, if any.
const char *VAPassName = Hints.vectorizeAnalysisPassName();
if (!VectorizeLoop && !InterleaveLoop) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 63436c79e9a98..1a0f19fe71e4d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1237,6 +1237,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
ExplicitVectorLength,
+ // Represents the incoming loop-invariant alias-mask.
+ IncomingAliasMask,
CalculateTripCountMinusVF,
// Increment the canonical IV separately for each unrolled part.
CanonicalIVIncrementForPart,
@@ -1275,8 +1277,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// part if it is scalar. In the latter case, the recipe will be removed
// during unrolling.
ExtractPenultimateElement,
- LogicalAnd, // Non-poison propagating logical And.
- LogicalOr, // Non-poison propagating logical Or.
+ LogicalAnd, // Non-poison propagating logical And.
+ LogicalOr, // Non-poison propagating logical Or.
+ NumActiveLanes, // Counts the number of active lanes in a mask.
// Add an offset in bytes (second operand) to a base pointer (first
// operand). Only generates scalar values (either for the first lane only or
// for all lanes, depending on its uses).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index a42b631cd3304..905162354e14e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -113,6 +113,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return IntegerType::get(Ctx, 1);
case VPInstruction::ExplicitVectorLength:
return Type::getIntNTy(Ctx, 32);
+ case VPInstruction::IncomingAliasMask:
+ return IntegerType::get(Ctx, 1);
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::Not:
case VPInstruction::CalculateTripCountMinusVF:
@@ -126,6 +128,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return inferScalarType(R->getOperand(1));
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
+ case VPInstruction::NumActiveLanes:
// Assume that the maximum possible number of elements in a vector fits
// within the index type for the default address space.
return DL.getIndexType(Ctx, 0);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e20d5d947ac54..3aefa8a909bd7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1430,13 +1430,19 @@ static void addBypassBranch(VPlan &Plan, VPBasicBlock *CheckBlockVPBB,
}
}
+void VPlanTransforms::attachCheckBlock(VPlan &Plan, VPValue *Cond,
+ VPBasicBlock *CheckBlock,
+ bool AddBranchWeights) {
+ insertCheckBlockBeforeVectorLoop(Plan, CheckBlock);
+ addBypassBranch(Plan, CheckBlock, Cond, AddBranchWeights);
+}
+
void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
BasicBlock *CheckBlock,
bool AddBranchWeights) {
VPValue *CondVPV = Plan.getOrAddLiveIn(Cond);
VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock(CheckBlock);
- insertCheckBlockBeforeVectorLoop(Plan, CheckBlockVPBB);
- addBypassBranch(Plan, CheckBlockVPBB, CondVPV, AddBranchWeights);
+ attachCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights);
}
/// Return an insert point in \p EntryVPBB after existing VPIRPhi,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 11a91dcd46867..069653dc5ba67 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -492,6 +492,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
switch (Opcode) {
case VPInstruction::StepVector:
case VPInstruction::VScale:
+ case VPInstruction::IncomingAliasMask:
return 0;
case Instruction::Alloca:
case Instruction::ExtractValue:
@@ -509,6 +510,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ResumeForEpilogue:
case VPInstruction::Reverse:
case VPInstruction::Unpack:
+ case VPInstruction::NumActiveLanes:
return 1;
case Instruction::ICmp:
case Instruction::FCmp:
@@ -655,6 +657,20 @@ Value *VPInstruction::generate(VPTransformState &State) {
{PredTy, ScalarTC->getType()},
{VIVElem0, ScalarTC}, nullptr, Name);
}
+ case VPInstruction::NumActiveLanes: {
+ Value *Op = State.get(getOperand(0));
+ auto *VecTy = cast<VectorType>(Op->getType());
+ assert(VecTy->getScalarSizeInBits() == 1 &&
+ "NumActiveLanes only implemented for i1 vectors");
+
+ Type *Ty = State.TypeAnalysis.inferScalarType(this);
+ Value *ZExt = Builder.CreateCast(
+ Instruction::ZExt, Op,
+ VectorType::get(Builder.getInt32Ty(), VecTy->getElementCount()));
+ Value *Count =
+ Builder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, ZExt);
+ return Builder.CreateCast(Instruction::ZExt, Count, Ty, "num.active.lanes");
+ }
case VPInstruction::FirstOrderRecurrenceSplice: {
// Generate code to combine the previous and current values in vector v3.
//
@@ -1317,7 +1333,8 @@ bool VPInstruction::isVectorToScalar() const {
getOpcode() == VPInstruction::LastActiveLane ||
getOpcode() == VPInstruction::ExtractLastActive ||
getOpcode() == VPInstruction::ComputeReductionResult ||
- getOpcode() == VPInstruction::AnyOf;
+ getOpcode() == VPInstruction::AnyOf ||
+ getOpcode() == VPInstruction::NumActiveLanes;
}
bool VPInstruction::isSingleScalar() const {
@@ -1394,6 +1411,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::IncomingAliasMask:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
@@ -1514,6 +1532,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ActiveLaneMask:
O << "active lane mask";
break;
+ case VPInstruction::IncomingAliasMask:
+ O << "incoming-alias-mask";
+ break;
case VPInstruction::ExplicitVectorLength:
O << "EXPLICIT-VECTOR-LENGTH";
break;
@@ -1601,6 +1622,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractLastActive:
O << "extract-last-active";
break;
+ case VPInstruction::NumActiveLanes:
+ O << "num-active-lanes";
+ break;
default:
O << Instruction::getOpcodeName(getOpcode());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32d89a34105a4..83de73537afd8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1191,7 +1191,7 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
}
}
-/// Remove redundant EpxandSCEVRecipes in \p Plan's entry block by replacing
+/// Remove redundant ExpandSCEVRecipes in \p Plan's entry block by replacing
/// them with already existing recipes expanding the same SCEV expression.
static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
DenseMap<const SCEV *, VPValue *> SCEV2VPV;
@@ -1205,7 +1205,11 @@ static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
const auto &[V, Inserted] = SCEV2VPV.try_emplace(ExpR->getSCEV(), ExpR);
if (Inserted)
continue;
+
ExpR->replaceAllUsesWith(V->second);
+ if (ExpR == Plan.getTripCount())
+ Plan.resetTripCount(V->second);
+
ExpR->eraseFromParent();
}
}
@@ -4955,8 +4959,8 @@ void VPlanTransforms::materializeVectorTripCount(
Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(TC);
VPBasicBlock::iterator InsertPt = VectorPHVPBB->begin();
if (auto *StepR = Step->getDefiningRecipe()) {
- assert(StepR->getParent() == VectorPHVPBB &&
- "Step must be defined in VectorPHVPBB");
+ assert(VPDominatorTree(Plan).dominates(StepR->getParent(), VectorPHVPBB) &&
+ "Step VPBB must dominate VectorPHVPBB");
// Insert after Step's definition to maintain valid def-use ordering.
InsertPt = std::next(StepR->getIterator());
}
@@ -5052,6 +5056,116 @@ void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
VFxUF.replaceAllUsesWith(MulByUF);
}
+void VPlanTransforms::attachAliasMaskToHeaderMask(VPlan &Plan) {
+ VPSingleDefRecipe *HeaderMask = vputils::findHeaderMask(Plan);
+ auto *HeaderMaskDef = HeaderMask->getDefiningRecipe();
+
+ VPBuilder Builder(Plan.getVectorPreheader());
+ auto *AliasMask = Builder.createNaryOp(VPInstruction::IncomingAliasMask, {});
+
+ if (HeaderMaskDef->isPhi())
+ Builder = VPBuilder(&*HeaderMaskDef->getParent()->getFirstNonPhi());
+ else
+ Builder = VPBuilder::getToInsertAfter(HeaderMaskDef);
+
+ // Update all existing users of the header mask to "HeaderMask & AliasMask".
+ auto *ClampedHeaderMask = Builder.createAnd(HeaderMask, AliasMask);
+ HeaderMask->replaceUsesWithIf(ClampedHeaderMask, [&](VPUser &U, unsigned) {
+ return dyn_cast<VPInstruction>(&U) != ClampedHeaderMask;
+ });
+}
+
+VPValue *
+VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
+ ArrayRef<PointerDiffInfo> DiffChecks) {
+ VPBuilder Builder(AliasCheckVPBB);
+ Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
+ Type *I64Ty = IntegerType::getInt64Ty(Plan.getContext());
+ Type *PtrTy = PointerType::getUnqual(Plan.getContext());
+
+ VPValue *IncomingAliasMask = vputils::findIncomingAliasMask(Plan);
+ assert(IncomingAliasMask && "Expected an alias mask!");
+
+ VPValue *AliasMask = nullptr;
+ for (PointerDiffInfo Check : DiffChecks) {
+ VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SrcStart);
+ VPValue *Sink =
+ vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SinkStart);
+
+ VPValue *SrcPtr =
+ Builder.createScalarCast(Instruction::CastOps::IntToPtr, Src, PtrTy,
+ DebugLoc::getCompilerGenerated());
+ VPValue *SinkPtr =
+ Builder.createScalarCast(Instruction::CastOps::IntToPtr, Sink, PtrTy,
+ DebugLoc::getCompilerGenerated());
+
+ VPWidenIntrinsicRecipe *WARMask = new VPWidenIntrinsicRecipe(
+ Intrinsic::loop_dependence_war_mask,
+ {SrcPtr, SinkPtr, Plan.getConstantInt(I64Ty, Check.AccessSize)}, I1Ty);
+ Builder.insert(WARMask);
+
+ if (AliasMask)
+ AliasMask = Builder.createAnd(AliasMask, WARMask);
+ else
+ AliasMask = WARMask;
+ }
+
+ Type *IVTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
+ VPValue *NumActive =
+ Builder.createNaryOp(VPInstruction::NumActiveLanes, {AliasMask});
+ VPValue *ClampedVF = Builder.createScalarZExtOrTrunc(
+ NumActive, IVTy, I64Ty, DebugLoc::getCompilerGenerated());
+
+ IncomingAliasMask->replaceAllUsesWith(AliasMask);
+
+ return ClampedVF;
+}
+
+void VPlanTransforms::materializeAliasMaskCheckBlock(
+ VPlan &Plan, ArrayRef<PointerDiffInfo> DiffChecks, bool HasBranchWeights) {
+ VPBasicBlock *ClampedVFCheck =
+ Plan.createVPBasicBlock("vector.clamped.vf.check");
+
+ VPValue *ClampedVF = materializeAliasMask(Plan, ClampedVFCheck, DiffChecks);
+ VPBuilder Builder(ClampedVFCheck);
+ DebugLoc DL = DebugLoc::getCompilerGenerated();
+ Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(Plan.getTripCount());
+
+ // Check the "ClampedVF" from the alias mask is larger than one.
+ VPValue *IsScalar =
+ Builder.createICmp(CmpInst::ICMP_ULE, ClampedVF,
+ Plan.getConstantInt(TCTy, 1), DL, "vf.is.scalar");
+
+ VPValue *TripCount = Plan.getTripCount();
+ VPValue *MaxUIntTripCount =
+ Plan.getConstantInt(cast<IntegerType>(TCTy)->getMask());
+ VPValue *DistanceToMax = Builder.createSub(MaxUIntTripCount, TripCount);
+
+ // For tail-folding: Don't execute the vector loop if (UMax - n) < ClampedVF.
+ // Note: The ClampedVF may not be a power-of-two. This means the loop exit
+ // condition (index.next == n.vec) may not be correct in the case of an
+ // overflow. The issue is `n.vec` could be zero due to an overflow, but
+ // index.next is not guaranteed to overflow to zero as the ClampedVF is not a
+ // power-of-two).
+ VPValue *TripCountCheck = Builder.createICmp(
+ ICmpInst::ICMP_ULT, DistanceToMax, ClampedVF, DL, "vf.step.overflow");
+
+ VPValue *Cond = Builder.createOr(IsScalar, TripCountCheck, DL);
+ attachCheckBlock(Plan, Cond, ClampedVFCheck, HasBranchWeights);
+
+ // Materialize the trip count early as this will add a use of (VFxUF) that
+ // needs to be replaced with the ClampedVF.
+ materializeVectorTripCount(Plan, Plan.getVectorPreheader(),
+ /*TailByMasking=*/true,
+ /*RequiresScalarEpilogue=*/false,
+ &Plan.getVFxUF());
+
+ assert(Plan.getConcreteUF() == 1 &&
+ "Clamped VF not supported with interleaving");
+ Plan.getVF().replaceAllUsesWith(ClampedVF);
+ Plan.getVFxUF().replaceAllUsesWith(ClampedVF);
+}
+
DenseMap<const SCEV *, Value *>
VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
SCEVExpander Expander(SE, "induction", /*PreserveLCSSA=*/false);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 75fc549167e03..61b345ac8053a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -196,6 +196,8 @@ struct VPlanTransforms {
/// Wrap runtime check block \p CheckBlock in a VPIRBB and \p Cond in a
/// VPValue and connect the block to \p Plan, using the VPValue as branch
/// condition.
+ static void attachCheckBlock(VPlan &Plan, VPValue *Cond,
+ VPBasicBlock *CheckBlock, bool AddBranchWeights);
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock,
bool AddBranchWeights);
@@ -434,6 +436,21 @@ struct VPlanTransforms {
static void materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
ElementCount VF);
+ /// Attaches the alias-mask to the existing header-mask.
+ static void attachAliasMaskToHeaderMask(VPlan &Plan);
+
+ /// Materializes within the \p AliasCheckVPBB block. Updates the header mask
+ /// of the loop to use the alias mask. Returns the clamped VF.
+ static VPValue *materializeAliasMask(VPlan &Plan,
+ VPBasicBlock *AliasCheckVPBB,
+ ArrayRef<PointerDiffInfo> DiffChecks);
+
+ /// Materializes the alias mask within a check block before the loop. The
+ /// vector loop will only be entered if the clamped VF from the alias mask
+ /// is not scalar.
+ static void materializeAliasMaskCheckBlock(
+ VPlan &Plan, ArrayRef<PointerDiffInfo> DiffChecks, bool HasBranchWeights);
+
/// Expand VPExpandSCEVRecipes in \p Plan's entry block. Each
/// VPExpandSCEVRecipe is replaced with a live-in wrapping the expanded IR
/// value. A mapping from SCEV expressions to their expanded IR value is
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 24adcea1040b5..81f4b215bf9b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -46,7 +46,8 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
if (U && !isa<Instruction>(U->getValue()))
return Plan.getOrAddLiveIn(U->getValue());
auto *Expanded = new VPExpandSCEVRecipe(Expr);
- Plan.getEntry()->appendRecipe(Expanded);
+ VPBasicBlock *EntryVPBB = Plan.getEntry();
+ EntryVPBB->insert(Expanded, EntryVPBB->getFirstNonPhi());
return Expanded;
}
@@ -80,8 +81,12 @@ bool vputils::isHeaderMask(const VPValue *V, const VPlan &Plan) {
}
auto MaskMatch = m_ICmp(m_VPValue(A), m_VPValue(B));
- return (match(V, m_CombineOr(MaskMatch, m_Reverse(MaskMatch)))) &&
- IsWideCanonicalIV(A) && B == Plan.getBackedgeTakenCount();
+ if (match(V, m_CombineOr(MaskMatch, m_Reverse(MaskMatch))))
+ return IsWideCanonicalIV(A) && B == Plan.getBackedgeTakenCount();
+
+ return match(
+ V, m_c_BinaryAnd(m_VPValue(),
+ m_VPInstruction<VPInstruction::IncomingAliasMask>()));
}
/// Returns true if \p R propagates poison from any operand to its result.
@@ -619,6 +624,13 @@ vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
}
VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
+ if (VPValue *AliasMask = findIncomingAliasMask(Plan)) {
+ assert(match(AliasMask->getSingleUser(),
+ m_c_BinaryAnd(m_VPValue(), m_Specific(AliasMask))) &&
+ "AliasMask must be only be used with the original header mask");
+ return cast<VPSingleDefRecipe>(AliasMask->getSingleUser());
+ }
+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
SmallVector<VPValue *> WideCanonicalIVs;
auto *WideCanonicalIV = vputils::findUserOf<VPWidenCanonicalIVRecipe>(
@@ -653,6 +665,15 @@ VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
HeaderMask = VPI;
}
}
+
+ for (VPRecipeBase &R : LoopRegion->getEntryBasicBlock()->phis()) {
+ auto *Def = cast<VPSingleDefRecipe>(&R);
+ if (vputils::isHeaderMask(Def, Plan)) {
+ assert(!HeaderMask && "Multiple header masks found?");
+ HeaderMask = Def;
+ }
+ }
+
return HeaderMask;
}
@@ -677,6 +698,13 @@ VPBlockUtils::blocksInSingleSuccessorChainBetween(VPBasicBlock *FirstBB,
return Blocks;
}
+VPValue *vputils::findIncomingAliasMask(const VPlan &Plan) {
+ for (VPRecipeBase &R : *const_cast<VPlan &>(Plan).getVectorPreheader())
+ if (match(&R, m_VPInstruction<VPInstruction::IncomingAliasMask>()))
+ return cast<VPInstruction>(&R);
+ return nullptr;
+}
+
bool VPBlockUtils::isHeader(const VPBlockBase *VPB,
const VPDominatorTree &VPDT) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2a4b8566d8475..fe9394eb8f96c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -169,10 +169,15 @@ VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
/// Collect the header mask with the pattern:
/// (ICMP_ULE, WideCanonicalIV, backedge-taken-count)
+/// Note: If alias masking is enabled this will find:
+/// (AND, HeaderMask, AliasMask)
/// TODO: Introduce explicit recipe for header-mask instead of searching
/// the header-mask pattern manually.
VPSingleDefRecipe *findHeaderMask(VPlan &Plan);
+/// Finds the incoming alias-mask within the vector preheader.
+VPValue *findIncomingAliasMask(const VPlan &Plan);
+
} // namespace vputils
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
new file mode 100644
index 0000000000000..c0cce7013ec8f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -0,0 +1,454 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -S -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize %s | FileCheck %s --check-prefix=CHECK-TF
+
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-TF-LABEL: define void @alias_mask(
+; CHECK-TF-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-TF-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-TF-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-TF-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK-TF: [[FOR_BODY_PREHEADER]]:
+; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
+; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-TF-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = and <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
+; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], <vscale x 16 x i8> splat (i8 1)
+; CHECK-TF-NEXT: [[TMP14:%.*]] = sdiv <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]], [[TMP13]]
+; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[TMP10]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP17:%.*]] = xor i1 [[TMP16]], true
+; CHECK-TF-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %div = sdiv i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %div, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %for.body, %entry
+ ret void
+}
+
+; Note: This test could emit a `llvm.loop.dependence.raw` mask to avoid creating
+; a dependency between the store and the load, but it is not necessary for
+; correctness.
+define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-TF-LABEL: define i32 @alias_mask_read_after_write(
+; CHECK-TF-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[C2:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-TF-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-TF-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-TF-NEXT: br i1 [[CMP19]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK-TF: [[FOR_BODY_PREHEADER]]:
+; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[C2]] to ptr
+; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[B1]] to ptr
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[TMP0]], ptr [[TMP1]], i64 4)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 4 x i1> [[ALIAS_MASK]] to <vscale x 4 x i32>
+; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP3]])
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-TF-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = and <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
+; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP11]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 2 [[TMP12]], <vscale x 4 x i1> [[TMP10]])
+; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP13]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_LOAD3]]
+; CHECK-TF-NEXT: [[TMP16]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
+; CHECK-TF-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+
+
+entry:
+ %cmp19 = icmp sgt i64 %n, 0
+ br i1 %cmp19, label %for.body, label %exit
+
+for.body: ; preds = %entry, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load.a = load i32, ptr %gep.a, align 2
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %load.a, ptr %gep.c, align 2
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %load.b = load i32, ptr %gep.b, align 2
+ %add = add i32 %load.a, %accum
+ %add2 = add i32 %add, %load.b
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %entry, %for.body
+ %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+ ret i32 %result
+}
+
+define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-TF-LABEL: define void @alias_mask_multiple(
+; CHECK-TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[A3:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-TF-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-TF-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-TF-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-TF-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK-TF: [[FOR_BODY_PREHEADER]]:
+; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[A3]] to ptr
+; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-TF-NEXT: [[TMP4:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-TF-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = and <vscale x 16 x i1> [[TMP2]], [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
+; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-TF-NEXT: [[TMP9:%.*]] = sub i64 -1, [[N]]
+; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP9]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-TF-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP14:%.*]] = and <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
+; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP15]], <vscale x 16 x i1> [[TMP14]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP16]], <vscale x 16 x i1> [[TMP14]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[TMP17:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]], [[WIDE_MASKED_LOAD]]
+; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP17]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[TMP14]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
+; CHECK-TF-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %add = add i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit: ; preds = %for.body, %entry
+ ret void
+}
+
+; Checks using a scalar outside the loop, with requires extracting the last
+; active element.
+define i8 @alias_masking_exit_value(ptr %ptrA, ptr %ptrB) {
+; CHECK-TF-LABEL: define i8 @alias_masking_exit_value(
+; CHECK-TF-SAME: ptr [[PTRA:%.*]], ptr [[PTRB:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[PTRA2:%.*]] = ptrtoaddr ptr [[PTRA]] to i64
+; CHECK-TF-NEXT: [[PTRB1:%.*]] = ptrtoaddr ptr [[PTRB]] to i64
+; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[PTRA2]] to ptr
+; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[PTRB1]] to ptr
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
+; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-TF-NEXT: [[TMP5:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i32
+; CHECK-TF-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-TF-NEXT: [[TMP7:%.*]] = mul i8 1, [[TMP6]]
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP7]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i32 [[TMP5]], 1
+; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i32 -1001, [[TMP5]]
+; CHECK-TF-NEXT: [[TMP8:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-TF-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1000)
+; CHECK-TF-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP12]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP13:%.*]] = and <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
+; CHECK-TF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[PTRA]], i32 [[INDEX]]
+; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[PTRB]], i32 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP14]], <vscale x 16 x i1> [[TMP13]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[TMP16:%.*]] = add <vscale x 16 x i8> [[VEC_IND]], [[WIDE_MASKED_LOAD]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP16]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[TMP13]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP5]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1000)
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[TMP19:%.*]] = xor <vscale x 16 x i1> [[TMP13]], splat (i1 true)
+; CHECK-TF-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP19]], i1 false)
+; CHECK-TF-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-TF-NEXT: [[TMP20:%.*]] = extractelement <vscale x 16 x i8> [[TMP16]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-TF-NEXT: br [[EXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %gepA = getelementptr inbounds i8, ptr %ptrA, i32 %iv
+ %gepB = getelementptr inbounds i8, ptr %ptrB, i32 %iv
+ %loadA = load i8, ptr %gepA
+ %iv.trunc = trunc i32 %iv to i8
+ %add = add i8 %iv.trunc, %loadA
+ store i8 %add, ptr %gepB
+ %iv.next = add nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %exit.value = phi i8 [ %add, %loop ]
+ ret i8 %exit.value
+}
+
+; Unsupported: Reversing the alias mask is not correct.
+define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC, i64 %n) {
+; CHECK-TF-LABEL: define void @alias_mask_reverse_iterate(
+; CHECK-TF-SAME: ptr noalias [[PTRA:%.*]], ptr [[PTRB:%.*]], ptr [[PTRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[PTRC2:%.*]] = ptrtoaddr ptr [[PTRC]] to i64
+; CHECK-TF-NEXT: [[PTRB1:%.*]] = ptrtoaddr ptr [[PTRB]] to i64
+; CHECK-TF-NEXT: [[IV_START:%.*]] = add i64 [[N]], -1
+; CHECK-TF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-TF: [[VECTOR_MEMCHECK]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-TF-NEXT: [[TMP2:%.*]] = sub i64 [[PTRB1]], [[PTRC2]]
+; CHECK-TF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
+; CHECK-TF-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 4
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[IV_START]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[IV_START]], [[INDEX]]
+; CHECK-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[PTRA]], i64 [[OFFSET_IDX]]
+; CHECK-TF-NEXT: [[TMP9:%.*]] = sub nuw nsw i64 [[TMP4]], 1
+; CHECK-TF-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], -1
+; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]]
+; CHECK-TF-NEXT: [[REVERSE:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[REVERSE]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[REVERSE3:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD]])
+; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[PTRB]], i64 [[OFFSET_IDX]]
+; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP10]]
+; CHECK-TF-NEXT: [[REVERSE4:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[REVERSE4]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[REVERSE6:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD5]])
+; CHECK-TF-NEXT: [[TMP14:%.*]] = add <vscale x 16 x i8> [[REVERSE6]], [[REVERSE3]]
+; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[PTRC]], i64 [[OFFSET_IDX]]
+; CHECK-TF-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP10]]
+; CHECK-TF-NEXT: [[REVERSE7:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[TMP14]])
+; CHECK-TF-NEXT: [[REVERSE8:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[REVERSE7]], ptr align 1 [[TMP16]], <vscale x 16 x i1> [[REVERSE8]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[IV_START]])
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br [[EXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+entry:
+ %iv.start = add nsw i64 %n, -1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ]
+ %gep.A = getelementptr inbounds i8, ptr %ptrA, i64 %iv
+ %loadA = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %ptrB, i64 %iv
+ %loadB = load i8, ptr %gep.B, align 1
+ %add = add i8 %loadB, %loadA
+ %gep.C = getelementptr inbounds i8, ptr %ptrC, i64 %iv
+ store i8 %add, ptr %gep.C, align 1
+ %iv.next = add nsw i64 %iv, -1
+ %ec = icmp eq i64 %iv.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Test taken from: scalable-first-order-recurrence.ll. Check we don't use
+; an alias-mask with first-order recurrences, as we cannot handle the
+; splice.right with the alias-mask/clamped VF yet.
+define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
+; CHECK-TF-LABEL: define i32 @recurrence_1(
+; CHECK-TF-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-TF-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-TF-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK-TF: [[FOR_PREHEADER]]:
+; CHECK-TF-NEXT: [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-TF-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-TF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-TF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-TF: [[VECTOR_MEMCHECK]]:
+; CHECK-TF-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-TF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-TF-NEXT: [[TMP6:%.*]] = add i64 [[B1]], -4
+; CHECK-TF-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[A2]]
+; CHECK-TF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
+; CHECK-TF-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-TF-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT: [[TMP14:%.*]] = mul nuw i32 [[TMP13]], 4
+; CHECK-TF-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1
+; CHECK-TF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP15]]
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP16:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP18]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP20]], ptr align 4 [[TMP19]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP2]])
+; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
+; CHECK-TF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-TF-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP23]], i1 false)
+; CHECK-TF-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-TF-NEXT: [[TMP24:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
+; CHECK-TF-NEXT: [[TMP25:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i64 [[TMP24]]
+; CHECK-TF-NEXT: [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT: [[TMP27:%.*]] = mul nuw i32 [[TMP26]], 4
+; CHECK-TF-NEXT: [[TMP28:%.*]] = sub i32 [[TMP27]], 1
+; CHECK-TF-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i32> [[VECTOR_RECUR]], i32 [[TMP28]]
+; CHECK-TF-NEXT: [[TMP30:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
+; CHECK-TF-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i32 [[TMP29]], i32 [[TMP25]]
+; CHECK-TF-NEXT: br [[FOR_EXIT:label %.*]]
+; CHECK-TF: [[SCALAR_PH]]:
+;
+
+entry:
+ br label %for.preheader
+
+for.preheader:
+ %pre_load = load i32, ptr %a
+ br label %scalar.body
+
+scalar.body:
+ %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
+ %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx32 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.next
+ %1 = load i32, ptr %arrayidx32
+ %arrayidx34 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
+ %add35 = add i32 %1, %0
+ store i32 %add35, ptr %arrayidx34
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.exit, label %scalar.body
+
+for.exit:
+ ret i32 %0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
new file mode 100644
index 0000000000000..88f88f5ffdb6c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -S -mattr=+v -mtriple riscv64 -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-tail-folding-style=data-with-evl -passes=loop-vectorize %s | FileCheck %s
+
+; Note: Alias masks are not supported with EVL at the moment.
+
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[C1]], [[B2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[CURRENT_ITERATION_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[CURRENT_ITERATION_IV]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[CURRENT_ITERATION_IV]]
+; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i8> @llvm.vp.merge.nxv16i8(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i8> [[VP_OP_LOAD]], <vscale x 16 x i8> splat (i8 1), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP7:%.*]] = sdiv <vscale x 16 x i8> [[VP_OP_LOAD3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[CURRENT_ITERATION_IV]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP7]], ptr align 1 [[TMP8]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[CURRENT_ITERATION_IV]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %div = sdiv i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %div, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
new file mode 100644
index 0000000000000..eb884af700b83
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+sve2 -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefixes=FINAL %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; FINAL-LABEL: VPlan for loop in 'alias_mask'
+; FINAL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4,vscale x 8,vscale x 16},UF={1}' {
+; FINAL-NEXT: Live-in ir<%n> = original trip-count
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<entry>:
+; FINAL-NEXT: IR %b2 = ptrtoaddr ptr %b to i64
+; FINAL-NEXT: IR %c1 = ptrtoaddr ptr %c to i64
+; FINAL-NEXT: Successor(s): vector.clamped.vf.check
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.clamped.vf.check:
+; FINAL-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = inttoptr ir<%b2> to ptr
+; FINAL-NEXT: EMIT-SCALAR vp<[[VP3:%[0-9]+]]> = inttoptr ir<%c1> to ptr
+; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP4:%[0-9]+]]> = call llvm.loop.dependence.war.mask(vp<[[VP2]]>, vp<[[VP3]]>, ir<1>)
+; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = num-active-lanes vp<[[VP4]]>
+; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP5]]>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP6]]>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP7]]>
+; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.ph:
+; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: Successor(s): vector.body
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.body:
+; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP9:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = and vp<[[VP9]]>, vp<[[VP4]]>
+; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP10]]>
+; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP10]]>
+; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
+; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP10]]>
+; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; FINAL-NEXT: Successor(s): middle.block, vector.body
+; FINAL-EMPTY:
+; FINAL-NEXT: middle.block:
+; FINAL-NEXT: Successor(s): ir-bb<exit>
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<exit>:
+; FINAL-NEXT: No successors
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<scalar.ph>:
+; FINAL-NEXT: Successor(s): ir-bb<for.body>
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<for.body>:
+; FINAL-NEXT: IR %iv = phi i64 [ 0, %scalar.ph ], [ %iv.next, %for.body ] (extra operand: ir<0> from ir-bb<scalar.ph>)
+; FINAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
+; FINAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
+; FINAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
+; FINAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
+; FINAL-NEXT: IR %add = add i8 %ld.b, %ld.a
+; FINAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
+; FINAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
+; FINAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
+; FINAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
+; FINAL-NEXT: No successors
+; FINAL-NEXT: }
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %ld.a = load i8, ptr %ptr.a, align 1
+ %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %ld.b = load i8, ptr %ptr.b, align 1
+ %add = add i8 %ld.b, %ld.a
+ %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %ptr.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
new file mode 100644
index 0000000000000..3724112123f95
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="attachAliasMaskToHeaderMask$" -S %s 2>&1 | FileCheck --check-prefix=INITIAL %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefix=FINAL %s
+
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; INITIAL-LABEL: VPlan for loop in 'alias_mask'
+; INITIAL: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; INITIAL-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; INITIAL-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; INITIAL-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; INITIAL-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
+; INITIAL-NEXT: Live-in ir<%n> = original trip-count
+; INITIAL-EMPTY:
+; INITIAL-NEXT: ir-bb<entry>:
+; INITIAL-NEXT: Successor(s): scalar.ph, vector.ph
+; INITIAL-EMPTY:
+; INITIAL-NEXT: vector.ph:
+; INITIAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = incoming-alias-mask
+; INITIAL-NEXT: Successor(s): vector loop
+; INITIAL-EMPTY:
+; INITIAL-NEXT: <x1> vector loop: {
+; INITIAL-NEXT: vector.body:
+; INITIAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; INITIAL-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; INITIAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP5]]>
+; INITIAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; INITIAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = and vp<[[VP7]]>, vp<[[VP4]]>
+; INITIAL-NEXT: Successor(s): vector.body.split
+; INITIAL-EMPTY:
+; INITIAL-NEXT: vector.body.split:
+; INITIAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; INITIAL-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.a>
+; INITIAL-NEXT: WIDEN ir<%ld.a> = load vp<[[VP9]]>, vp<[[VP8]]>
+; INITIAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; INITIAL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.b>
+; INITIAL-NEXT: WIDEN ir<%ld.b> = load vp<[[VP10]]>, vp<[[VP8]]>
+; INITIAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
+; INITIAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, ir<%iv>
+; INITIAL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.c>
+; INITIAL-NEXT: WIDEN store vp<[[VP11]]>, ir<%add>, vp<[[VP8]]>
+; INITIAL-NEXT: CLONE ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; INITIAL-NEXT: CLONE ir<%exitcond.not> = icmp eq ir<%iv.next>, ir<%n>
+; INITIAL-NEXT: Successor(s): vector.latch
+; INITIAL-EMPTY:
+; INITIAL-NEXT: vector.latch:
+; INITIAL-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
+; INITIAL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; INITIAL-NEXT: No successors
+; INITIAL-NEXT: }
+; INITIAL-NEXT: Successor(s): middle.block
+; INITIAL-EMPTY:
+; INITIAL-NEXT: middle.block:
+; INITIAL-NEXT: EMIT branch-on-cond ir<true>
+; INITIAL-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; INITIAL-EMPTY:
+; INITIAL-NEXT: ir-bb<exit>:
+; INITIAL-NEXT: No successors
+; INITIAL-EMPTY:
+; INITIAL-NEXT: scalar.ph:
+; INITIAL-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<%n>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; INITIAL-NEXT: Successor(s): ir-bb<for.body>
+; INITIAL-EMPTY:
+; INITIAL-NEXT: ir-bb<for.body>:
+; INITIAL-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; INITIAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
+; INITIAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
+; INITIAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
+; INITIAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
+; INITIAL-NEXT: IR %add = add i8 %ld.b, %ld.a
+; INITIAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
+; INITIAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
+; INITIAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
+; INITIAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
+; INITIAL-NEXT: No successors
+; INITIAL-NEXT: }
+;
+; FINAL-LABEL: VPlan for loop in 'alias_mask'
+; FINAL: VPlan 'Final VPlan for VF={4},UF={1}' {
+; FINAL-NEXT: Live-in ir<%n> = original trip-count
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<entry>:
+; FINAL-NEXT: IR %b2 = ptrtoaddr ptr %b to i64
+; FINAL-NEXT: IR %c1 = ptrtoaddr ptr %c to i64
+; FINAL-NEXT: Successor(s): vector.clamped.vf.check
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.clamped.vf.check:
+; FINAL-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = inttoptr ir<%b2> to ptr
+; FINAL-NEXT: EMIT-SCALAR vp<[[VP3:%[0-9]+]]> = inttoptr ir<%c1> to ptr
+; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP4:%[0-9]+]]> = call llvm.loop.dependence.war.mask(vp<[[VP2]]>, vp<[[VP3]]>, ir<1>)
+; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = num-active-lanes vp<[[VP4]]>
+; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP5]]>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP6]]>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP7]]>
+; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.ph:
+; FINAL-NEXT: EMIT vp<%trip.count.minus.1> = sub ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = sub vp<[[VP5]]>, ir<1>
+; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP9]]>
+; FINAL-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<%n.mod.vf>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; FINAL-NEXT: Successor(s): vector.body
+; FINAL-EMPTY:
+; FINAL-NEXT: vector.body:
+; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
+; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ule vp<[[VP11]]>, vp<[[VP10]]>
+; FINAL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = and vp<[[VP12]]>, vp<[[VP4]]>
+; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP13]]>
+; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP13]]>
+; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
+; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP13]]>
+; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; FINAL-NEXT: Successor(s): middle.block, vector.body
+; FINAL-EMPTY:
+; FINAL-NEXT: middle.block:
+; FINAL-NEXT: Successor(s): ir-bb<exit>
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<exit>:
+; FINAL-NEXT: No successors
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<scalar.ph>:
+; FINAL-NEXT: Successor(s): ir-bb<for.body>
+; FINAL-EMPTY:
+; FINAL-NEXT: ir-bb<for.body>:
+; FINAL-NEXT: IR %iv = phi i64 [ 0, %scalar.ph ], [ %iv.next, %for.body ] (extra operand: ir<0> from ir-bb<scalar.ph>)
+; FINAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
+; FINAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
+; FINAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
+; FINAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
+; FINAL-NEXT: IR %add = add i8 %ld.b, %ld.a
+; FINAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
+; FINAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
+; FINAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
+; FINAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
+; FINAL-NEXT: No successors
+; FINAL-NEXT: }
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %ld.a = load i8, ptr %ptr.a, align 1
+ %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %ld.b = load i8, ptr %ptr.b, align 1
+ %add = add i8 %ld.b, %ld.a
+ %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %ptr.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index d9eaab8d9a000..deda7d439cdfc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -481,12 +481,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
-; CHECK-NEXT: IR %inc = add i64 %div, 1
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP4:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
+; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
+; CHECK-NEXT: IR %inc = add i64 %div, 1
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
new file mode 100644
index 0000000000000..398f46a5d58cf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-tail-folding-style=data -force-vector-width=4 %s | FileCheck %s
+
+define void @test(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[SRC2]] to ptr
+; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[DST1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP2]], ptr [[TMP3]], i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i32
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i32 [[TMP7]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 -1, [[UMAX3]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[TMP7]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[UMAX3]], [[TMP10]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP7]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX1]], i32 [[UMAX3]])
+; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[INDEX1]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP12]], <4 x i1> [[MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX1]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP13]], <4 x i1> [[MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX1]], [[TMP7]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_CLAMPED_VF_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv.next = add i32 %iv, 1
+ %gep.src = getelementptr i32, ptr %src, i32 %iv
+ %val = load i32, ptr %gep.src, align 4
+ %gep.dst = getelementptr i32, ptr %dst, i32 %iv
+ store i32 %val, ptr %gep.dst, align 4
+ %cond = icmp ult i32 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
new file mode 100644
index 0000000000000..2726b32911432
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+
+; Note: First order recurrences are not supported with alias-masking.
+define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
+; CHECK-LABEL: define i32 @first_order_recurrence(
+; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: br label %[[FOR_PREHEADER:.*]]
+; CHECK: [[FOR_PREHEADER]]:
+; CHECK-NEXT: [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[B1]], -4
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], [[A2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 16
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP5]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[TMP8]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP10]], ptr align 4 [[TMP9]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[WIDE_MASKED_LOAD]], i64 [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[VECTOR_RECUR]], i32 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP14]]
+; CHECK-NEXT: br [[FOR_EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.preheader
+
+for.preheader:
+ %pre_load = load i32, ptr %a
+ br label %scalar.body
+
+scalar.body:
+ %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
+ %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx32 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.next
+ %1 = load i32, ptr %arrayidx32
+ %arrayidx34 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
+ %add35 = add i32 %1, %0
+ store i32 %add35, ptr %arrayidx34
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.exit, label %scalar.body
+
+for.exit:
+ ret i32 %0
+}
+
+; This loop uses bounds checks (not diff checks), so can't use an alias mask.
+define void @uses_bounds_checks(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @uses_bounds_checks(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[N]]
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[C]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT3]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i32> [[WIDE_MASKED_LOAD4]] to <4 x i8>
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i8> [[TMP4]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP5]], ptr align 1 [[TMP6]], <4 x i1> [[TMP1]]), !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %ptr.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %ptr.a, align 1
+ %ptr.b = getelementptr inbounds nuw i32, ptr %b, i64 %iv
+ %load.b = load i32, ptr %ptr.b, align 4
+ %b.trunc = trunc i32 %load.b to i8
+ %add = add i8 %b.trunc, %load.a
+ %ptr.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv
+ store i8 %add, ptr %ptr.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
new file mode 100644
index 0000000000000..4c66c1e6c1834
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -0,0 +1,382 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=2 -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -epilogue-vectorization-force-VF=2 -force-vector-interleave=2 -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
+
+; Note: -force-vector-interleave and -epilogue-vectorization-force-VF does not
+; change the results as alias-masking is not supported with interleaving or
+; epilogue vectorization.
+
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP9]], <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv <8 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP13]], ptr align 1 [[TMP14]], <8 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %div = sdiv i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %div, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Alias mask created via combining multiple dependence masks.
+define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask_multiple(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A3:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[A3]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = and <8 x i1> [[TMP2]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP7]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP9]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT4]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP12:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP12]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP14]], <8 x i1> [[TMP13]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP15]], <8 x i1> [[TMP13]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP16:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD6]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP16]], ptr align 1 [[TMP17]], <8 x i1> [[TMP13]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %add = add i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Alias masking + a simple add reduction.
+define i32 @alias_mask_with_reduction(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define i32 @alias_mask_with_reduction(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP12]], ptr align 1 [[TMP13]], <8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[TMP12]] to <8 x i32>
+; CHECK-NEXT: [[TMP15]] = add <8 x i32> [[VEC_PHI]], [[TMP14]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP17:%.*]] = select <8 x i1> [[TMP9]], <8 x i32> [[TMP15]], <8 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP17]])
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %reduce = phi i32 [ 0, %entry ], [ %reduce.next, %for.body ]
+ %ptr.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %ld.a = load i8, ptr %ptr.a, align 1
+ %ptr.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %ld.b = load i8, ptr %ptr.b, align 1
+ %add = add i8 %ld.b, %ld.a
+ %ptr.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv
+ store i8 %add, ptr %ptr.c, align 1
+ %ext.add = zext i8 %add to i32
+ %reduce.next = add nuw nsw i32 %reduce, %ext.add
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %reduce.next
+}
+
+define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, ptr addrspace(1) %b, ptr addrspace(1) %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask_non_default_address_space(
+; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr addrspace(1) [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr addrspace(1) [[C]] to i64
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p1(ptr addrspace(1) align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p1(ptr addrspace(1) align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP9]], <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv <8 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p1(<8 x i8> [[TMP13]], ptr addrspace(1) align 1 [[TMP14]], <8 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ %cmp11 = icmp sgt i64 %n, 0
+ br i1 %cmp11, label %for.body, label %exit
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %iv
+ %load.a = load i8, ptr addrspace(1) %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr addrspace(1) %b, i64 %iv
+ %load.b = load i8, ptr addrspace(1) %gep.b, align 1
+ %div = sdiv i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr addrspace(1) %c, i64 %iv
+ store i8 %div, ptr addrspace(1) %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Test alias mask with a known trip-count that would be one iteration of the full VF.
+define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
+; CHECK-LABEL: define void @alias_mask_known_trip_count(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 -8, [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 7, [[TMP6]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <8 x i64> [[VEC_IV]], splat (i64 6)
+; CHECK-NEXT: [[TMP8:%.*]] = and <8 x i1> [[TMP7]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP9]], <8 x i1> [[TMP8]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP8]], <8 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <8 x i1> [[TMP8]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %ptr.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %ptr.a, align 1
+ %ptr.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %ptr.b, align 1
+ %add = add i8 %load.b, %load.a
+ %ptr.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv
+ store i8 %add, ptr %ptr.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 7
+ br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 41b2f6f9e62d8..c88bb2553fb19 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -430,8 +430,8 @@ define i64 @ivopt_widen_ptr_indvar_1(ptr noalias %a, i64 %stride, i64 %n) {
;
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_1(
; STRIDED-NEXT: entry:
-; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
+; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; STRIDED: vector.ph:
@@ -515,8 +515,8 @@ define i64 @ivopt_widen_ptr_indvar_2(ptr noalias %a, i64 %stride, i64 %n) {
;
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_2(
; STRIDED-NEXT: entry:
-; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
+; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; STRIDED: vector.ph:
@@ -620,8 +620,8 @@ define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
;
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_3(
; STRIDED-NEXT: entry:
-; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
+; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; STRIDED: vector.ph:
@@ -702,10 +702,10 @@ define void @strided_ptr_iv_runtime_stride(ptr %pIn, ptr %pOut, i32 %nCols, i32
; STRIDED-NEXT: entry:
; STRIDED-NEXT: [[PIN2:%.*]] = ptrtoaddr ptr [[PIN:%.*]] to i64
; STRIDED-NEXT: [[POUT1:%.*]] = ptrtoaddr ptr [[POUT:%.*]] to i64
-; STRIDED-NEXT: [[TMP0:%.*]] = zext i32 [[NCOLS:%.*]] to i64
-; STRIDED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
; STRIDED-NEXT: [[TMP1:%.*]] = sext i32 [[STRIDE:%.*]] to i64
; STRIDED-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP1]], 2
+; STRIDED-NEXT: [[TMP10:%.*]] = zext i32 [[NCOLS:%.*]] to i64
+; STRIDED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP10]], i64 1)
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; STRIDED: vector.scevcheck:
diff --git a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
new file mode 100644
index 0000000000000..7aeb4cf5deb8a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S %s | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+define void @test(ptr %base_a, ptr %base_b, i32 %ntypes) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[BASE_A:%.*]], ptr [[BASE_B:%.*]], i32 [[NTYPES:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[NTYPES]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[N:%.*]] = zext nneg i32 [[NTYPES]] to i64
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[N]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IND:%.*]] = add <4 x i64> [[BROADCAST_SPLAT6]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[BASE_A]], <4 x i64> [[VEC_IND1]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [8 x i8], ptr [[BASE_B]], i64 [[TMP20]]
+; CHECK-NEXT: call void @llvm.masked.store.v4p0.p0(<4 x ptr> [[TMP3]], ptr align 8 [[TMP23]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND1]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp sgt i32 %ntypes, 0
+ br i1 %cmp, label %loop.preheader, label %exit
+
+loop.preheader:
+ %n = zext nneg i32 %ntypes to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %loop.preheader ], [ %iv.next, %loop ]
+ %offset = phi i64 [ 0, %loop.preheader ], [ %offset.next, %loop ]
+ %gep_a = getelementptr inbounds [4 x i8], ptr %base_a, i64 %offset
+ %gep_b = getelementptr inbounds [8 x i8], ptr %base_b, i64 %iv
+ store ptr %gep_a, ptr %gep_b, align 8
+ %offset.next = add nuw nsw i64 %offset, %n
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
index 84cdaa869395e..afb9a71137fef 100644
--- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
@@ -205,10 +205,15 @@ define void @expand_diff_scev_unknown(ptr %dst, i1 %invar.c, i32 %step) mustprog
; CHECK-NEXT: br i1 [[INVAR_C]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1]]
; CHECK: [[LOOP_2_PREHEADER]]:
; CHECK-NEXT: [[IV_1_LCSSA:%.*]] = phi i32 [ [[IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i32 2, [[STEP]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[IV_1_LCSSA]], [[TMP0]]
+; CHECK-NEXT: [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP12]], i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[INDVAR]], -1
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP3]], -1
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[SMAX1]], [[TMP14]]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[IV_1_LCSSA]], [[STEP]]
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0)
; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[STEP]], -2
-; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[INDVAR]], -1
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[SMAX]], [[TMP4]]
; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP5]], i32 1)
@@ -217,11 +222,6 @@ define void @expand_diff_scev_unknown(ptr %dst, i1 %invar.c, i32 %step) mustprog
; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[STEP]], i32 1)
; CHECK-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP7]], [[UMAX]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP6]], [[TMP8]]
-; CHECK-NEXT: [[TMP16:%.*]] = sub i32 2, [[STEP]]
-; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[IV_1_LCSSA]], [[TMP16]]
-; CHECK-NEXT: [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP12]], i32 0)
-; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP3]], -1
-; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[SMAX1]], [[TMP14]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP15]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
>From d7002464344eaa4c14183f6d116c2c1ab689fd06 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 23 Mar 2026 16:05:03 +0000
Subject: [PATCH 2/7] Update assert
Format code
Fixups
Add test for expensive alias-mask
Handle uniforms
Add assert
Rebase fixups
More uniform tests
---
.../Vectorize/LoopVectorizationLegality.h | 4 +-
.../Vectorize/LoopVectorizationLegality.cpp | 13 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 78 ++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +-
.../AArch64/alias-mask-uniforms.ll | 277 ++++++++++++++++++
.../LoopVectorize/AArch64/alias-mask.ll | 2 +-
.../AArch64/expensive-alias-masking.ll | 85 ++++++
.../Transforms/LoopVectorize/alias-mask.ll | 160 +++++-----
8 files changed, 519 insertions(+), 103 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..c5d0da5f1269e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -403,12 +403,12 @@ class LoopVectorizationLegality {
/// Returns true if value V is uniform across \p VF lanes, when \p VF is
/// provided, and otherwise if \p V is invariant across all loop iterations.
- bool isUniform(Value *V, ElementCount VF) const;
+ bool isUniform(Value *V, std::optional<ElementCount> VF) const;
/// A uniform memory op is a load or store which accesses the same memory
/// location on all \p VF lanes, if \p VF is provided and otherwise if the
/// memory location is invariant.
- bool isUniformMemOp(Instruction &I, ElementCount VF) const;
+ bool isUniformMemOp(Instruction &I, std::optional<ElementCount> VF) const;
/// Returns the information that we collected about runtime memory check.
const RuntimePointerChecking *getRuntimePointerChecking() const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index bee08eeba9927..e330b3f6214be 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -579,12 +579,13 @@ class SCEVAddRecForUniformityRewriter
} // namespace
-bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
+bool LoopVectorizationLegality::isUniform(
+ Value *V, std::optional<ElementCount> VF) const {
if (isInvariant(V))
return true;
- if (VF.isScalable())
+ if (!VF || VF->isScalable())
return false;
- if (VF.isScalar())
+ if (VF->isScalar())
return true;
// Since we rely on SCEV for uniformity, if the type is not SCEVable, it is
@@ -596,7 +597,7 @@ bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
// Rewrite AddRecs in TheLoop to step by VF and check if the expression for
// lane 0 matches the expressions for all other lanes.
- unsigned FixedVF = VF.getKnownMinValue();
+ unsigned FixedVF = VF->getKnownMinValue();
const SCEV *FirstLaneExpr =
SCEVAddRecForUniformityRewriter::rewrite(S, *SE, FixedVF, 0, TheLoop);
if (isa<SCEVCouldNotCompute>(FirstLaneExpr))
@@ -612,8 +613,8 @@ bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
});
}
-bool LoopVectorizationLegality::isUniformMemOp(Instruction &I,
- ElementCount VF) const {
+bool LoopVectorizationLegality::isUniformMemOp(
+ Instruction &I, std::optional<ElementCount> VF) const {
Value *Ptr = getLoadStorePointerOperand(&I);
if (!Ptr)
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 576fa50be565a..4bb941b966bf2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -814,6 +814,8 @@ enum EpilogueLowering {
CM_EpilogueNotAllowedFoldTail
};
+enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
+
/// LoopVectorizationCostModel - estimates the expected speedups due to
/// vectorization.
/// In many cases vectorization is not profitable. This can happen because of
@@ -1233,10 +1235,14 @@ class LoopVectorizationCostModel {
}
void tryToEnablePartialAliasMasking() {
- assert(!IsPartialAliasMaskingEnabled && "Alias masking already enabled!");
assert(foldTailByMasking() && "Expected tail folding to be enabled!");
assert(!foldTailWithEVL() &&
"Did not expect to enable alias masking with EVL!");
+ assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
+
+ // Assume we fail to enable alias masking (in case we early exit).
+ PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
+
// Note: FixedOrderRecurrences are not supported yet as we cannot handle
// the required `splice.right` with the alias-mask.
if (!ForcePartialAliasingVectorization ||
@@ -1251,10 +1257,18 @@ class LoopVectorizationCostModel {
if (!DiffChecks || DiffChecks->empty())
return;
+ [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
+ return any_of(CB->args(), [](Value const *Arg) {
+ return Arg->getType()->isPointerTy();
+ });
+ };
+
for (BasicBlock *BB : TheLoop->blocks()) {
for (Instruction &I : *BB) {
if (!isa<LoadInst, StoreInst>(I)) {
- assert(isa<CallInst>(I) || !I.mayReadOrWriteMemory());
+ [[maybe_unused]] auto *Call = dyn_cast<CallInst>(&I);
+ assert((!I.mayReadOrWriteMemory() || Call && !HasPointerArgs(Call)) &&
+ "Skipped unexpected memory access");
continue;
}
@@ -1265,16 +1279,18 @@ class LoopVectorizationCostModel {
// alias mask is not correct (or necessary). When combined with
// tail-folding the active lane mask should only be reversed where the
// alias-mask is true.
- if (Legal->isConsecutivePtr(ScalarTy, Ptr) != 1)
+ if (Legal->isConsecutivePtr(ScalarTy, Ptr) == -1)
return;
}
}
- IsPartialAliasMaskingEnabled = true;
+ PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
}
/// Returns true if all loop blocks should have partial aliases masked.
- bool maskPartialAliasing() const { return IsPartialAliasMaskingEnabled; }
+ bool maskPartialAliasing() const {
+ return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
+ }
/// Returns true if the use of wide lane masks is requested and the loop is
/// using tail-folding with a lane mask for control flow.
@@ -1368,6 +1384,33 @@ class LoopVectorizationCostModel {
/// VF selection state independent of cost-modeling decisions.
VFSelectionContext &Config;
+ /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
+ /// account if alias-masking is enabled. We consider the VF to be unknown when
+ /// alias masking.
+ bool isUniform(Value *V, ElementCount VF) const {
+ // With alias-masking our runtime VF is [2, VF] (and not necessarily a
+ // power-of-two). Something that is uniform for VF may not be for the full
+ // range.
+ assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
+ "alias-mask status must be decided already");
+ return Legal->isUniform(V, PartialAliasMaskingStatus ==
+ AliasMaskingStatus::Disabled
+ ? std::optional(VF)
+ : std::nullopt);
+ }
+
+ /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
+ /// account if alias-masking is enabled. We consider the VF to be unknown when
+ /// alias masking.
+ bool isUniformMemOp(Instruction &I, ElementCount VF) const {
+ assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
+ "alias-mask status must be decided already");
+ return Legal->isUniformMemOp(I, PartialAliasMaskingStatus ==
+ AliasMaskingStatus::Disabled
+ ? std::optional(VF)
+ : std::nullopt);
+ }
+
/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
@@ -1417,8 +1460,8 @@ class LoopVectorizationCostModel {
/// Control finally chosen tail folding style.
TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
- /// True if partial alias masking is enabled.
- bool IsPartialAliasMaskingEnabled = false;
+ /// If partial alias masking is enabled/disabled or not decided.
+ AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
/// A map holding scalar costs for different vectorization factors. The
/// presence of a cost for an instruction in the mapping indicates that the
@@ -1672,6 +1715,9 @@ class GeneratedRTChecks {
}
const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
+ // TODO: We need to estimate the cost of alias-masking in
+ // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
+ // alias-mask is generated later in VPlan.
if (RtPtrChecking.Need && !LoopUsesAliasMasking) {
auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
@@ -2753,7 +2799,7 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (Iter != Uniforms.end() && !Iter->second.contains(I))
return false;
}
- if (!Legal->isUniformMemOp(*I, VF))
+ if (!isUniformMemOp(*I, VF))
return false;
if (isa<LoadInst>(I))
// Loading the same address always produces the same result - at least
@@ -2830,7 +2876,7 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
// If the pointer can be proven to be uniform, always add it to the
// worklist.
- if (isa<Instruction>(Ptr) && Legal->isUniform(Ptr, VF))
+ if (isa<Instruction>(Ptr) && isUniform(Ptr, VF))
AddToWorklistIfAllowed(cast<Instruction>(Ptr));
if (IsUniformMemOpUse(&I))
@@ -2923,6 +2969,12 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
FixedScalableVFPair
LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
+ // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
+ scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
+ if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
+ PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
+ });
+
// For outer loops, use simple type-based heuristic VF. No cost model or
// memory dependence analysis is available.
if (!TheLoop->isInnermost()) {
@@ -4408,7 +4460,7 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
InstructionCost
LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
ElementCount VF) {
- assert(Legal->isUniformMemOp(*I, VF));
+ assert(isUniformMemOp(*I, VF));
Type *ValTy = getLoadStoreType(I);
Type *PtrTy = getLoadStorePointerOperand(I)->getType();
@@ -4449,7 +4501,7 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
Value *Ptr = getLoadStorePointerOperand(I);
Type *PtrTy = Ptr->getType();
- if (!Legal->isUniform(Ptr, VF))
+ if (!isUniform(Ptr, VF))
PtrTy = toVectorTy(PtrTy, VF);
unsigned IID = I->getOpcode() == Instruction::Load
@@ -4784,7 +4836,7 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (isa<StoreInst>(&I) && isScalarWithPredication(&I, VF))
NumPredStores++;
- if (Legal->isUniformMemOp(I, VF)) {
+ if (isUniformMemOp(I, VF)) {
auto IsLegalToScalarize = [&]() {
if (!VF.isScalable())
// Scalarization of fixed length vectors "just works".
@@ -4953,7 +5005,7 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
InstWidening Decision = getWideningDecision(I, VF);
if (!isPredicatedInst(I) &&
(Decision == CM_Widen || Decision == CM_Widen_Reverse ||
- (!Legal->isUniformMemOp(*I, VF) && Decision == CM_Scalarize))) {
+ (!isUniformMemOp(*I, VF) && Decision == CM_Scalarize))) {
// Scalarize a widened load of address or update the cost of a scalar
// load of an address.
setWideningDecision(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1a0f19fe71e4d..53a2fd64fdb9a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1237,7 +1237,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
ExplicitVectorLength,
- // Represents the incoming loop-invariant alias-mask.
+ // Represents the incoming loop-invariant alias-mask. All memory accesses
+ // in the loop must stay within the active lanes.
IncomingAliasMask,
CalculateTripCountMinusVF,
// Increment the canonical IV separately for each unrolled part.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
new file mode 100644
index 0000000000000..042146df0f3d3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+
+; Tests `%div = sdiv i64 %iv, 64` (and its use for a load) is not considered
+; to be uniform with partial alias masking.
+define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
+; CHECK-LABEL: define void @vf_dependent_uniform(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
+; CHECK-NEXT: [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE10:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE10]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv <4 x i64> [[VEC_IND]], splat (i64 64)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[MASK]], i32 0
+; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP14]], i32 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[MASK]], i32 1
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i32 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[MASK]], i32 2
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; CHECK: [[PRED_LOAD_IF7]]:
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP25]], align 8
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i32 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; CHECK: [[PRED_LOAD_CONTINUE8]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[MASK]], i32 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10]]
+; CHECK: [[PRED_LOAD_IF9]]:
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP10]], i32 3
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr [[TMP31]], align 8
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP32]], i32 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
+; CHECK: [[PRED_LOAD_CONTINUE10]]:
+; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP33]], %[[PRED_LOAD_IF9]] ]
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP35]], <4 x i1> [[MASK]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP36:%.*]] = add <4 x i64> [[TMP34]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP36]], ptr align 8 [[TMP37]], <4 x i1> [[MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %div = sdiv i64 %iv, 64
+ %gep = getelementptr inbounds i64, ptr %p, i64 %div
+ %ld = load i64, ptr %gep, align 8
+ %gep2 = getelementptr inbounds i64, ptr %p.in, i64 %iv
+ %ld2 = load i64, ptr %gep2, align 8
+ %val = add i64 %ld, %ld2
+ %store.gep = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %store.gep, align 8
+ %iv.next = add nsw i64 %iv, 1
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %loop, label %exit
+exit:
+ ret void
+}
+
+define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
+; CHECK-LABEL: define void @uniform_load(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
+; CHECK-NEXT: [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP22]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[MASK]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i64> [[TMP17]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP19]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ld = load i64, ptr %p, align 8
+ %gep = getelementptr inbounds i64, ptr %p.in, i64 %iv
+ %ld2 = load i64, ptr %gep, align 8
+ %val = add i64 %ld, %ld2
+ %store.gep = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %store.gep, align 8
+ %iv.next = add nsw i64 %iv, 1
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %loop, label %exit
+exit:
+ ret void
+}
+
+define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
+; CHECK-LABEL: define void @uniform_store(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
+; CHECK-NEXT: [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE10:.*]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP10]], <4 x i1> [[MASK]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i64> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[MASK]], i32 0
+; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP11]], i32 0
+; CHECK-NEXT: store i64 [[TMP13]], ptr [[P]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[MASK]], i32 1
+; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP11]], i32 1
+; CHECK-NEXT: store i64 [[TMP15]], ptr [[P]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[MASK]], i32 2
+; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP11]], i32 2
+; CHECK-NEXT: store i64 [[TMP17]], ptr [[P]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[MASK]], i32 3
+; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10]]
+; CHECK: [[PRED_STORE_IF9]]:
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP11]], i32 3
+; CHECK-NEXT: store i64 [[TMP19]], ptr [[P]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; CHECK: [[PRED_STORE_CONTINUE10]]:
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP11]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i64, ptr %p.in, i64 %iv
+ %ld = load i64, ptr %gep, align 8
+ %val = add i64 %ld, %ld
+ store i64 %val, ptr %p, align 8
+ %store.gep = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %store.gep, align 8
+ %iv.next = add nsw i64 %iv, 1
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %loop, label %exit
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index c0cce7013ec8f..065bb58f227e1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -318,7 +318,7 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC,
; CHECK-TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[IV_START]], [[INDEX]]
; CHECK-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[PTRA]], i64 [[OFFSET_IDX]]
; CHECK-TF-NEXT: [[TMP9:%.*]] = sub nuw nsw i64 [[TMP4]], 1
-; CHECK-TF-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], -1
+; CHECK-TF-NEXT: [[TMP10:%.*]] = sub i64 0, [[TMP9]]
; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]]
; CHECK-TF-NEXT: [[REVERSE:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[REVERSE]], <vscale x 16 x i8> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll b/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
new file mode 100644
index 0000000000000..7044e652df8da
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
@@ -0,0 +1,85 @@
+; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
+; RUN: cat %t | FileCheck %s -check-prefix=CHECK-ALIAS-MASKING-REMARKS
+; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
+; RUN: cat %t | FileCheck %s -check-prefix=CHECK-DIFF-CHECKS-REMARKS
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; This loop has four store pointers and eight load pointers. It requires 38
+; diff checks (no store alias = (4*3)/2 = 6 checks, no store alias with load =
+; 4 * 8 = 32 checks, for a total of 38 checks). The loops trip count is low (33).
+;
+; Diff checks are determined to unprofitable due to the high number of checks
+; and low trip count.
+;
+; With alias-masking we do vectorize this loop as the cost of setting up the
+; alias-mask is not factored into the vectorization cost.
+;
+; TODO: Cost the alias-mask when using -force-partial-aliasing-vectorization.
+
+; CHECK-DIFF-CHECKS-REMARKS: loop not vectorized
+
+; CHECK-ALIAS-MASKING-REMARKS: vectorized loop (vectorization width: vscale x 4, interleaved count: 1)
+
+define void @expensive_runtime_checks(ptr %0, ptr %1, ptr %2) {
+entry:
+ %5 = load ptr, ptr %1, align 8
+ %6 = load ptr, ptr %2, align 8
+ %7 = load ptr, ptr %0, align 8
+ %8 = getelementptr inbounds nuw i8, ptr %1, i64 8
+ %9 = load ptr, ptr %8, align 8
+ %10 = getelementptr inbounds nuw i8, ptr %2, i64 8
+ %11 = load ptr, ptr %10, align 8
+ %12 = getelementptr inbounds nuw i8, ptr %0, i64 8
+ %13 = load ptr, ptr %12, align 8
+ %14 = getelementptr inbounds nuw i8, ptr %1, i64 16
+ %15 = load ptr, ptr %14, align 8
+ %16 = getelementptr inbounds nuw i8, ptr %2, i64 16
+ %17 = load ptr, ptr %16, align 8
+ %18 = getelementptr inbounds nuw i8, ptr %0, i64 16
+ %19 = load ptr, ptr %18, align 8
+ %20 = getelementptr inbounds nuw i8, ptr %1, i64 24
+ %21 = load ptr, ptr %20, align 8
+ %22 = getelementptr inbounds nuw i8, ptr %2, i64 24
+ %23 = load ptr, ptr %22, align 8
+ %24 = getelementptr inbounds nuw i8, ptr %0, i64 24
+ %25 = load ptr, ptr %24, align 8
+ br label %loop
+
+loop:
+ %27 = phi i64 [ 0, %entry ], [ %52, %loop ]
+ %28 = getelementptr inbounds nuw [4 x i8], ptr %5, i64 %27
+ %29 = load i32, ptr %28, align 4
+ %30 = getelementptr inbounds nuw [4 x i8], ptr %6, i64 %27
+ %31 = load i32, ptr %30, align 4
+ %32 = add nsw i32 %31, %29
+ %33 = getelementptr inbounds nuw [4 x i8], ptr %7, i64 %27
+ store i32 %32, ptr %33, align 4
+ %34 = getelementptr inbounds nuw [4 x i8], ptr %9, i64 %27
+ %35 = load i32, ptr %34, align 4
+ %36 = getelementptr inbounds nuw [4 x i8], ptr %11, i64 %27
+ %37 = load i32, ptr %36, align 4
+ %38 = add nsw i32 %37, %35
+ %39 = getelementptr inbounds nuw [4 x i8], ptr %13, i64 %27
+ store i32 %38, ptr %39, align 4
+ %40 = getelementptr inbounds nuw [4 x i8], ptr %15, i64 %27
+ %41 = load i32, ptr %40, align 4
+ %42 = getelementptr inbounds nuw [4 x i8], ptr %17, i64 %27
+ %43 = load i32, ptr %42, align 4
+ %44 = add nsw i32 %43, %41
+ %45 = getelementptr inbounds nuw [4 x i8], ptr %19, i64 %27
+ store i32 %44, ptr %45, align 4
+ %46 = getelementptr inbounds nuw [4 x i8], ptr %21, i64 %27
+ %47 = load i32, ptr %46, align 4
+ %48 = getelementptr inbounds nuw [4 x i8], ptr %23, i64 %27
+ %49 = load i32, ptr %48, align 4
+ %50 = add nsw i32 %49, %47
+ %51 = getelementptr inbounds nuw [4 x i8], ptr %25, i64 %27
+ store i32 %50, ptr %51, align 4
+ %52 = add nuw nsw i64 %27, 1
+ %53 = icmp eq i64 %52, 33
+ br i1 %53, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index 4c66c1e6c1834..97e10c61d18c7 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=2 -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -epilogue-vectorization-force-VF=2 -force-vector-interleave=2 -force-vector-width=8 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -epilogue-vectorization-force-VF=2 -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
; Note: -force-vector-interleave and -epilogue-vectorization-force-VF does not
; change the results as alias-masking is not supported with interleaving or
@@ -20,9 +20,9 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
@@ -35,24 +35,24 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP10]], <4 x i1> [[TMP9]], <4 x i8> poison)
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP9]], <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> splat (i8 1)
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv <8 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i8> [[WIDE_MASKED_LOAD]], <4 x i8> splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv <4 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP13]], ptr align 1 [[TMP14]], <8 x i1> [[TMP9]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP13]], ptr align 1 [[TMP14]], <4 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -96,13 +96,13 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[A3]] to ptr
; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[B2]] to ptr
; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = and <8 x i1> [[TMP2]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP7]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i1> [[TMP2]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 -1, [[N]]
@@ -115,23 +115,23 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT4]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT: [[TMP12:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP13:%.*]] = and <8 x i1> [[TMP12]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP12:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = and <4 x i1> [[TMP12]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP14]], <8 x i1> [[TMP13]], <8 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP14]], <4 x i1> [[TMP13]], <4 x i8> poison)
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP15]], <8 x i1> [[TMP13]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP16:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD6]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP15]], <4 x i1> [[TMP13]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD6]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP16]], ptr align 1 [[TMP17]], <8 x i1> [[TMP13]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP16]], ptr align 1 [[TMP17]], <4 x i1> [[TMP13]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -171,9 +171,9 @@ define i32 @alias_mask_with_reduction(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
@@ -186,32 +186,32 @@ define i32 @alias_mask_with_reduction(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP10]], <4 x i1> [[TMP9]], <4 x i8> poison)
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP12:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP12]], ptr align 1 [[TMP13]], <8 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[TMP12]] to <8 x i32>
-; CHECK-NEXT: [[TMP15]] = add <8 x i32> [[VEC_PHI]], [[TMP14]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP12]], ptr align 1 [[TMP13]], <4 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
+; CHECK-NEXT: [[TMP15]] = add <4 x i32> [[VEC_PHI]], [[TMP14]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP17:%.*]] = select <8 x i1> [[TMP9]], <8 x i32> [[TMP15]], <8 x i32> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP17]])
+; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP15]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP17]])
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
;
@@ -251,9 +251,9 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
@@ -266,24 +266,24 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT3]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP9:%.*]] = and <8 x i1> [[TMP8]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p1(ptr addrspace(1) align 1 [[TMP10]], <8 x i1> [[TMP9]], <8 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p1(ptr addrspace(1) align 1 [[TMP10]], <4 x i1> [[TMP9]], <4 x i8> poison)
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p1(ptr addrspace(1) align 1 [[TMP11]], <8 x i1> [[TMP9]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP9]], <8 x i8> [[WIDE_MASKED_LOAD]], <8 x i8> splat (i8 1)
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv <8 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p1(ptr addrspace(1) align 1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i8> [[WIDE_MASKED_LOAD]], <4 x i8> splat (i8 1)
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv <4 x i8> [[WIDE_MASKED_LOAD5]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p1(<8 x i8> [[TMP13]], ptr addrspace(1) align 1 [[TMP14]], <8 x i1> [[TMP9]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p1(<4 x i8> [[TMP13]], ptr addrspace(1) align 1 [[TMP14]], <4 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -323,9 +323,9 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i1> [[ALIAS_MASK]] to <8 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 -8, [[NUM_ACTIVE_LANES]]
@@ -339,18 +339,18 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <8 x i64> [[VEC_IV]], splat (i64 6)
-; CHECK-NEXT: [[TMP8:%.*]] = and <8 x i1> [[TMP7]], [[ALIAS_MASK]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IV]], splat (i64 6)
+; CHECK-NEXT: [[TMP8:%.*]] = and <4 x i1> [[TMP7]], [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP9]], <8 x i1> [[TMP8]], <8 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP9]], <4 x i1> [[TMP8]], <4 x i8> poison)
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP10]], <8 x i1> [[TMP8]], <8 x i8> poison)
-; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP10]], <4 x i1> [[TMP8]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <8 x i1> [[TMP8]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP11]], ptr align 1 [[TMP12]], <4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
>From 0c54594da7a104cec9c0f3204d567e56b675e425 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 23 Apr 2026 12:58:03 +0000
Subject: [PATCH 3/7] Rebase fixups
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 +++
.../AArch64/alias-mask-uniforms.ll | 42 +++++++++----------
.../LoopVectorize/AArch64/alias-mask.ll | 20 ++++-----
.../AArch64/expensive-alias-masking.ll | 4 +-
.../RISCV/alias-mask-force-evl.ll | 2 +-
.../AArch64/vplan-printing-alias-mask.ll | 2 +-
.../VPlan/vplan-printing-alias-mask.ll | 12 +++---
.../alias-mask-data-tail-folding-style.ll | 5 +--
.../alias-mask-negative-tests.ll | 4 +-
.../Transforms/LoopVectorize/alias-mask.ll | 6 +--
.../remove-redundant-trip-count-scev.ll | 2 +-
11 files changed, 53 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 81f4b215bf9b8..d13a5203b3a50 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -762,6 +762,11 @@ VPInstruction *vputils::findCanonicalIVIncrement(VPlan &Plan) {
if (!UF.isMaterialized())
return Step == &UF;
+ // Alias masking: step is number of active lanes of a dependence mask.
+ if (match(Step, m_ZExtOrTruncOrSelf(
+ m_VPInstruction<VPInstruction::NumActiveLanes>())))
+ return true;
+
unsigned ConcreteUF = Plan.getConcreteUF();
// Fixed VF: step is just the concrete UF.
if (match(Step, m_SpecificInt(ConcreteUF)))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
index 042146df0f3d3..90036a35543eb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
; Tests `%div = sdiv i64 %iv, 64` (and its use for a load) is not considered
; to be uniform with partial alias masking.
@@ -40,43 +40,43 @@ define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n)
; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT: [[TMP10:%.*]] = sdiv <4 x i64> [[VEC_IND]], splat (i64 64)
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[MASK]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP14]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP14]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[MASK]], i32 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[MASK]], i64 1
; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP10]], i64 1
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_CONTINUE6]]:
; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[MASK]], i32 2
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[MASK]], i64 2
; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP10]], i64 2
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP24]]
; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP25]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i32 2
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i64 2
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
; CHECK: [[PRED_LOAD_CONTINUE8]]:
; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[MASK]], i32 3
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[MASK]], i64 3
; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10]]
; CHECK: [[PRED_LOAD_IF9]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP10]], i32 3
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP10]], i64 3
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP30]]
; CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr [[TMP31]], align 8
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP32]], i32 3
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP32]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
; CHECK: [[PRED_LOAD_CONTINUE10]]:
; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP33]], %[[PRED_LOAD_IF9]] ]
@@ -221,31 +221,31 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP10]], <4 x i1> [[MASK]], <4 x i64> poison)
; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i64> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[MASK]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP11]], i64 0
; CHECK-NEXT: store i64 [[TMP13]], ptr [[P]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[MASK]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[MASK]], i64 1
; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP11]], i64 1
; CHECK-NEXT: store i64 [[TMP15]], ptr [[P]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[MASK]], i32 2
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[MASK]], i64 2
; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP11]], i64 2
; CHECK-NEXT: store i64 [[TMP17]], ptr [[P]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[MASK]], i32 3
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[MASK]], i64 3
; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10]]
; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP11]], i32 3
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i64> [[TMP11]], i64 3
; CHECK-NEXT: store i64 [[TMP19]], ptr [[P]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
; CHECK: [[PRED_STORE_CONTINUE10]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index 065bb58f227e1..1eb3f2920f565 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize %s | FileCheck %s --check-prefix=CHECK-TF
+; RUN: opt -S -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -tail-folding-policy=must-fold-tail %s | FileCheck %s --check-prefix=CHECK-TF
define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-LABEL: define void @alias_mask(
@@ -40,7 +40,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[TMP10]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP17:%.*]] = xor i1 [[TMP16]], true
; CHECK-TF-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
@@ -113,7 +113,7 @@ define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n)
; CHECK-TF-NEXT: [[TMP16]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]]
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
@@ -190,7 +190,7 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP17]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[TMP14]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
; CHECK-TF-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
@@ -260,7 +260,7 @@ define i8 @alias_masking_exit_value(ptr %ptrA, ptr %ptrB) {
; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP16]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[TMP13]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP5]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1000)
-; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -325,18 +325,16 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC,
; CHECK-TF-NEXT: [[REVERSE3:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD]])
; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[PTRB]], i64 [[OFFSET_IDX]]
; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP10]]
-; CHECK-TF-NEXT: [[REVERSE4:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[REVERSE4]], <vscale x 16 x i8> poison)
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[REVERSE]], <vscale x 16 x i8> poison)
; CHECK-TF-NEXT: [[REVERSE6:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_LOAD5]])
; CHECK-TF-NEXT: [[TMP14:%.*]] = add <vscale x 16 x i8> [[REVERSE6]], [[REVERSE3]]
; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[PTRC]], i64 [[OFFSET_IDX]]
; CHECK-TF-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP10]]
; CHECK-TF-NEXT: [[REVERSE7:%.*]] = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> [[TMP14]])
-; CHECK-TF-NEXT: [[REVERSE8:%.*]] = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[REVERSE7]], ptr align 1 [[TMP16]], <vscale x 16 x i1> [[REVERSE8]])
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[REVERSE7]], ptr align 1 [[TMP16]], <vscale x 16 x i1> [[REVERSE]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[IV_START]])
-; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
@@ -410,7 +408,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP20]], ptr align 4 [[TMP19]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP2]])
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
; CHECK-TF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll b/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
index 7044e652df8da..3d1a3e7662979 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/expensive-alias-masking.ll
@@ -1,6 +1,6 @@
-; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
+; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -force-partial-aliasing-vectorization -tail-folding-policy=must-fold-tail %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
; RUN: cat %t | FileCheck %s -check-prefix=CHECK-ALIAS-MASKING-REMARKS
-; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
+; RUN: opt -S -disable-output -mattr=+sve2 -passes=loop-vectorize -tail-folding-policy=must-fold-tail %s -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize 2>%t
; RUN: cat %t | FileCheck %s -check-prefix=CHECK-DIFF-CHECKS-REMARKS
target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
index 88f88f5ffdb6c..bfdcf245e1785 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/alias-mask-force-evl.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -mattr=+v -mtriple riscv64 -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-tail-folding-style=data-with-evl -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -mattr=+v -mtriple riscv64 -force-partial-aliasing-vectorization -tail-folding-policy=must-fold-tail -force-tail-folding-style=data-with-evl -passes=loop-vectorize %s | FileCheck %s
; Note: Alias masks are not supported with EVL at the moment.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index eb884af700b83..079ccce94ce9a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-vectorize -mattr=+sve2 -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefixes=FINAL %s
+; RUN: opt -passes=loop-vectorize -mattr=+sve2 -force-partial-aliasing-vectorization -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefixes=FINAL %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index 3724112123f95..eccd4c30c84f0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="attachAliasMaskToHeaderMask$" -S %s 2>&1 | FileCheck --check-prefix=INITIAL %s
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefix=FINAL %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="attachAliasMaskToHeaderMask$" -S %s 2>&1 | FileCheck --check-prefix=INITIAL %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefix=FINAL %s
define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-LABEL: VPlan for loop in 'alias_mask'
@@ -19,8 +19,9 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Successor(s): vector loop
; INITIAL-EMPTY:
; INITIAL-NEXT: <x1> vector loop: {
+; INITIAL-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; INITIAL-EMPTY:
; INITIAL-NEXT: vector.body:
-; INITIAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; INITIAL-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; INITIAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP5]]>
; INITIAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
@@ -50,14 +51,13 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Successor(s): middle.block
; INITIAL-EMPTY:
; INITIAL-NEXT: middle.block:
-; INITIAL-NEXT: EMIT branch-on-cond ir<true>
-; INITIAL-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; INITIAL-NEXT: Successor(s): ir-bb<exit>
; INITIAL-EMPTY:
; INITIAL-NEXT: ir-bb<exit>:
; INITIAL-NEXT: No successors
; INITIAL-EMPTY:
; INITIAL-NEXT: scalar.ph:
-; INITIAL-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<%n>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; INITIAL-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb<entry> ]
; INITIAL-NEXT: Successor(s): ir-bb<for.body>
; INITIAL-EMPTY:
; INITIAL-NEXT: ir-bb<for.body>:
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
index 398f46a5d58cf..99435ea97b439 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-partial-aliasing-vectorization -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-tail-folding-style=data -force-vector-width=4 %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-partial-aliasing-vectorization -tail-folding-policy=prefer-fold-tail -force-tail-folding-style=data -force-vector-width=4 %s | FileCheck %s
define void @test(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test(
@@ -47,10 +47,9 @@ define void @test(ptr %src, ptr %dst, i32 %n) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_CLAMPED_VF_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[IV]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
index 2726b32911432..7c076d7dd726a 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
; Note: First order recurrences are not supported with alias-masking.
define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
@@ -52,7 +52,7 @@ define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b,
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[WIDE_MASKED_LOAD]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[VECTOR_RECUR]], i32 3
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[VECTOR_RECUR]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP14]]
; CHECK-NEXT: br [[FOR_EXIT:label %.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index 97e10c61d18c7..14d3c943b917a 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
-; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -prefer-predicate-over-epilogue=predicate-dont-vectorize -epilogue-vectorization-force-VF=2 -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -epilogue-vectorization-force-VF=2 -force-vector-interleave=2 -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
; Note: -force-vector-interleave and -epilogue-vectorization-force-VF does not
; change the results as alias-masking is not supported with interleaving or
diff --git a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
index 7aeb4cf5deb8a..7a38b3ca7cc23 100644
--- a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
>From f79c42a53f3e5e92581a2a331d38fb8ee0e34f95 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 24 Apr 2026 09:26:07 +0000
Subject: [PATCH 4/7] Rebase fixups
---
llvm/lib/Analysis/VectorUtils.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 +----
.../AArch64/alias-mask-uniforms.ll | 12 ++----
.../LoopVectorize/AArch64/alias-mask.ll | 29 +++----------
.../AArch64/vplan-printing-alias-mask.ll | 32 +++++++-------
.../VPlan/vplan-printing-alias-mask.ll | 42 +++++++++----------
.../alias-mask-data-tail-folding-style.ll | 4 +-
.../alias-mask-negative-tests.ll | 6 +--
.../Transforms/LoopVectorize/alias-mask.ll | 24 +++--------
9 files changed, 55 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 635e34af0b6fa..fb2f4f14f7489 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -196,6 +196,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::ucmp:
case Intrinsic::scmp:
case Intrinsic::vector_extract:
+ case Intrinsic::loop_dependence_war_mask:
return OpdIdx == -1 || OpdIdx == 0;
case Intrinsic::modf:
case Intrinsic::sincos:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 83de73537afd8..bc78e8a1de77c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5081,7 +5081,6 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
VPBuilder Builder(AliasCheckVPBB);
Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
Type *I64Ty = IntegerType::getInt64Ty(Plan.getContext());
- Type *PtrTy = PointerType::getUnqual(Plan.getContext());
VPValue *IncomingAliasMask = vputils::findIncomingAliasMask(Plan);
assert(IncomingAliasMask && "Expected an alias mask!");
@@ -5091,17 +5090,11 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SrcStart);
VPValue *Sink =
vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SinkStart);
-
- VPValue *SrcPtr =
- Builder.createScalarCast(Instruction::CastOps::IntToPtr, Src, PtrTy,
- DebugLoc::getCompilerGenerated());
- VPValue *SinkPtr =
- Builder.createScalarCast(Instruction::CastOps::IntToPtr, Sink, PtrTy,
- DebugLoc::getCompilerGenerated());
+ Type *AddrType = VPTypeAnalysis(Plan).inferScalarType(Src);
VPWidenIntrinsicRecipe *WARMask = new VPWidenIntrinsicRecipe(
Intrinsic::loop_dependence_war_mask,
- {SrcPtr, SinkPtr, Plan.getConstantInt(I64Ty, Check.AccessSize)}, I1Ty);
+ {Src, Sink, Plan.getConstantInt(AddrType, Check.AccessSize)}, I1Ty);
Builder.insert(WARMask);
if (AliasMask)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
index 90036a35543eb..b76fb0dcca492 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -12,9 +12,7 @@ define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n)
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -122,9 +120,7 @@ define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -191,9 +187,7 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[P_IN2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[P_OUT1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 8)
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index 1eb3f2920f565..3b8a67b8ca337 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -12,9 +12,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF: [[FOR_BODY_PREHEADER]]:
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -83,9 +81,7 @@ define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n)
; CHECK-TF: [[FOR_BODY_PREHEADER]]:
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[C2]] to ptr
-; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[B1]] to ptr
-; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1(ptr [[TMP0]], ptr [[TMP1]], i64 4)
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1.i64(i64 [[C2]], i64 [[B1]], i64 4)
; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 4 x i1> [[ALIAS_MASK]] to <vscale x 4 x i32>
; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP3]])
; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -159,12 +155,8 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF: [[FOR_BODY_PREHEADER]]:
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[A3]] to ptr
-; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-TF-NEXT: [[TMP3:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-TF-NEXT: [[TMP4:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-TF-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
+; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
+; CHECK-TF-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = and <vscale x 16 x i1> [[TMP2]], [[TMP5]]
; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
@@ -229,9 +221,7 @@ define i8 @alias_masking_exit_value(ptr %ptrA, ptr %ptrB) {
; CHECK-TF-NEXT: [[PTRB1:%.*]] = ptrtoaddr ptr [[PTRB]] to i64
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = inttoptr i64 [[PTRA2]] to ptr
-; CHECK-TF-NEXT: [[TMP1:%.*]] = inttoptr i64 [[PTRB1]] to ptr
-; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[PTRA2]], i64 [[PTRB1]], i64 1)
; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -415,14 +405,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
; CHECK-TF-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
; CHECK-TF-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP23]], i1 false)
; CHECK-TF-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-TF-NEXT: [[TMP24:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
-; CHECK-TF-NEXT: [[TMP25:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i64 [[TMP24]]
-; CHECK-TF-NEXT: [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-TF-NEXT: [[TMP27:%.*]] = mul nuw i32 [[TMP26]], 4
-; CHECK-TF-NEXT: [[TMP28:%.*]] = sub i32 [[TMP27]], 1
-; CHECK-TF-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i32> [[VECTOR_RECUR]], i32 [[TMP28]]
-; CHECK-TF-NEXT: [[TMP30:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
-; CHECK-TF-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i32 [[TMP29]], i32 [[TMP25]]
+; CHECK-TF-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i32> [[TMP18]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-TF-NEXT: br [[FOR_EXIT:label %.*]]
; CHECK-TF: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 079ccce94ce9a..530ccf7bf8e57 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -15,15 +15,13 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): vector.clamped.vf.check
; FINAL-EMPTY:
; FINAL-NEXT: vector.clamped.vf.check:
-; FINAL-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = inttoptr ir<%b2> to ptr
-; FINAL-NEXT: EMIT-SCALAR vp<[[VP3:%[0-9]+]]> = inttoptr ir<%c1> to ptr
-; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP4:%[0-9]+]]> = call llvm.loop.dependence.war.mask(vp<[[VP2]]>, vp<[[VP3]]>, ir<1>)
-; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = num-active-lanes vp<[[VP4]]>
-; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP5]]>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = sub ir<-1>, ir<%n>
-; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP6]]>, vp<[[VP5]]>
-; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP7]]>
+; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP2:%[0-9]+]]> = call llvm.loop.dependence.war.mask(ir<%b2>, ir<%c1>, ir<1>)
+; FINAL-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = num-active-lanes vp<[[VP2]]>
+; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP3]]>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP4]]>, vp<[[VP3]]>
+; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP5]]>
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
@@ -32,19 +30,19 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP9:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = and vp<[[VP9]]>, vp<[[VP4]]>
+; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP7:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = and vp<[[VP7]]>, vp<[[VP2]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP10]]>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP8]]>
; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP10]]>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP8]]>
; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
-; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP10]]>
-; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP5]]>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP8]]>
+; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP3]]>
; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not vp<%active.lane.mask.next>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP9]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index eccd4c30c84f0..13040befa57c0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -84,41 +84,39 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): vector.clamped.vf.check
; FINAL-EMPTY:
; FINAL-NEXT: vector.clamped.vf.check:
-; FINAL-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = inttoptr ir<%b2> to ptr
-; FINAL-NEXT: EMIT-SCALAR vp<[[VP3:%[0-9]+]]> = inttoptr ir<%c1> to ptr
-; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP4:%[0-9]+]]> = call llvm.loop.dependence.war.mask(vp<[[VP2]]>, vp<[[VP3]]>, ir<1>)
-; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = num-active-lanes vp<[[VP4]]>
-; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP5]]>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = sub ir<-1>, ir<%n>
-; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP6]]>, vp<[[VP5]]>
-; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP7]]>
+; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP2:%[0-9]+]]> = call llvm.loop.dependence.war.mask(ir<%b2>, ir<%c1>, ir<1>)
+; FINAL-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = num-active-lanes vp<[[VP2]]>
+; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP3]]>, ir<1>
+; FINAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP4]]>, vp<[[VP3]]>
+; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP5]]>
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
; FINAL-NEXT: EMIT vp<%trip.count.minus.1> = sub ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = sub vp<[[VP5]]>, ir<1>
-; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP9]]>
-; FINAL-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = sub vp<[[VP3]]>, ir<1>
+; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP7]]>
+; FINAL-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, vp<[[VP3]]>
; FINAL-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<%n.mod.vf>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
-; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ule vp<[[VP11]]>, vp<[[VP10]]>
-; FINAL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = and vp<[[VP12]]>, vp<[[VP4]]>
+; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<[[VP9]]>, vp<[[VP8]]>
+; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = and vp<[[VP10]]>, vp<[[VP2]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP13]]>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP11]]>
; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP13]]>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP11]]>
; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
-; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP13]]>
-; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP5]]>
-; FINAL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP11]]>
+; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP3]]>
+; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP12]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
index 99435ea97b439..a3d519a7304d4 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
@@ -15,9 +15,7 @@ define void @test(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[SRC2]] to ptr
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[DST1]] to ptr
-; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP2]], ptr [[TMP3]], i64 4)
+; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[SRC2]], i64 [[DST1]], i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP6]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
index 7c076d7dd726a..4df5a50948c81 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
@@ -50,11 +50,7 @@ define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b,
; CHECK-NEXT: [[TMP12:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[WIDE_MASKED_LOAD]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[VECTOR_RECUR]], i64 3
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP14]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP8]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br [[FOR_EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index 14d3c943b917a..e9c27ce38a3db 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -18,9 +18,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -94,12 +92,8 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[A3]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP3]], ptr [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i1> [[TMP2]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i32>
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
@@ -169,9 +163,7 @@ define i32 @alias_mask_with_reduction(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -249,9 +241,7 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
@@ -321,9 +311,7 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = inttoptr i64 [[B2]] to ptr
-; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i64 [[C1]] to ptr
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1(ptr [[TMP0]], ptr [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
>From a13c74b5e00636096f445d84ea27e61e7131ef14 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 6 May 2026 15:48:55 +0000
Subject: [PATCH 5/7] Rebase fixups
---
.../lib/Transforms/Vectorize/VPlanConstruction.cpp | 8 ++++----
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 5 +++--
.../LoopVectorize/AArch64/alias-mask-uniforms.ll | 14 ++++++++------
4 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 3aefa8a909bd7..b18454756e4b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1430,9 +1430,9 @@ static void addBypassBranch(VPlan &Plan, VPBasicBlock *CheckBlockVPBB,
}
}
-void VPlanTransforms::attachCheckBlock(VPlan &Plan, VPValue *Cond,
- VPBasicBlock *CheckBlock,
- bool AddBranchWeights) {
+void VPlanTransforms::attachVPCheckBlock(VPlan &Plan, VPValue *Cond,
+ VPBasicBlock *CheckBlock,
+ bool AddBranchWeights) {
insertCheckBlockBeforeVectorLoop(Plan, CheckBlock);
addBypassBranch(Plan, CheckBlock, Cond, AddBranchWeights);
}
@@ -1442,7 +1442,7 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
bool AddBranchWeights) {
VPValue *CondVPV = Plan.getOrAddLiveIn(Cond);
VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock(CheckBlock);
- attachCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights);
+ attachVPCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights);
}
/// Return an insert point in \p EntryVPBB after existing VPIRPhi,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bc78e8a1de77c..99da530d88201 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5144,7 +5144,7 @@ void VPlanTransforms::materializeAliasMaskCheckBlock(
ICmpInst::ICMP_ULT, DistanceToMax, ClampedVF, DL, "vf.step.overflow");
VPValue *Cond = Builder.createOr(IsScalar, TripCountCheck, DL);
- attachCheckBlock(Plan, Cond, ClampedVFCheck, HasBranchWeights);
+ attachVPCheckBlock(Plan, Cond, ClampedVFCheck, HasBranchWeights);
// Materialize the trip count early as this will add a use of (VFxUF) that
// needs to be replaced with the ClampedVF.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 61b345ac8053a..b60f633acd0f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -196,8 +196,9 @@ struct VPlanTransforms {
/// Wrap runtime check block \p CheckBlock in a VPIRBB and \p Cond in a
/// VPValue and connect the block to \p Plan, using the VPValue as branch
/// condition.
- static void attachCheckBlock(VPlan &Plan, VPValue *Cond,
- VPBasicBlock *CheckBlock, bool AddBranchWeights);
+ static void attachVPCheckBlock(VPlan &Plan, VPValue *Cond,
+ VPBasicBlock *CheckBlock,
+ bool AddBranchWeights);
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock,
bool AddBranchWeights);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
index b76fb0dcca492..d012e99724d01 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -124,6 +124,8 @@ define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
@@ -140,9 +142,7 @@ define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[P]], align 8
@@ -154,6 +154,7 @@ define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP19]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -191,6 +192,8 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
@@ -207,9 +210,7 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE10:.*]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE10]] ]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
@@ -246,6 +247,7 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP11]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From b740f1017ef623e5a9168629cd4fa50cff8dba3c Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 6 May 2026 18:49:41 +0000
Subject: [PATCH 6/7] Fixups
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 7 +-
.../LoopVectorize/AArch64/alias-mask.ll | 178 +-----------------
.../VPlan/vplan-printing-alias-mask.ll | 35 ++--
.../alias-mask-negative-tests.ll | 101 +++++++---
.../Transforms/LoopVectorize/alias-mask.ll | 129 +++----------
.../LoopVectorize/cast-induction.ll | 4 +-
.../remove-redundant-trip-count-scev.ll | 5 +-
8 files changed, 139 insertions(+), 326 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4bb941b966bf2..4f5e946b59c61 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1245,8 +1245,11 @@ class LoopVectorizationCostModel {
// Note: FixedOrderRecurrences are not supported yet as we cannot handle
// the required `splice.right` with the alias-mask.
+ // TODO: Reductions are not supported as values outside the "ClampedVF" are
+ // not masked out for the final horizontal reductions.
if (!ForcePartialAliasingVectorization ||
- !Legal->getFixedOrderRecurrences().empty())
+ !Legal->getFixedOrderRecurrences().empty() ||
+ !Legal->getReductionVars().empty())
return;
const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
@@ -7235,6 +7238,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *Cond = vputils::findHeaderMask(*Plan);
NewExitingVPV =
Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", *PhiR);
+
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
return match(&U,
m_VPInstruction<VPInstruction::ComputeReductionResult>());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 99da530d88201..53c5d33b4071f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5080,8 +5080,8 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
ArrayRef<PointerDiffInfo> DiffChecks) {
VPBuilder Builder(AliasCheckVPBB);
Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
- Type *I64Ty = IntegerType::getInt64Ty(Plan.getContext());
+ VPTypeAnalysis TypeInfo(Plan);
VPValue *IncomingAliasMask = vputils::findIncomingAliasMask(Plan);
assert(IncomingAliasMask && "Expected an alias mask!");
@@ -5090,7 +5090,7 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SrcStart);
VPValue *Sink =
vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SinkStart);
- Type *AddrType = VPTypeAnalysis(Plan).inferScalarType(Src);
+ Type *AddrType = TypeInfo.inferScalarType(Src);
VPWidenIntrinsicRecipe *WARMask = new VPWidenIntrinsicRecipe(
Intrinsic::loop_dependence_war_mask,
@@ -5107,7 +5107,8 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
VPValue *NumActive =
Builder.createNaryOp(VPInstruction::NumActiveLanes, {AliasMask});
VPValue *ClampedVF = Builder.createScalarZExtOrTrunc(
- NumActive, IVTy, I64Ty, DebugLoc::getCompilerGenerated());
+ NumActive, IVTy, TypeInfo.inferScalarType(NumActive),
+ DebugLoc::getCompilerGenerated());
IncomingAliasMask->replaceAllUsesWith(AliasMask);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index 3b8a67b8ca337..68b77544899ce 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -7,9 +7,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: [[ENTRY:.*:]]
; CHECK-TF-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
; CHECK-TF-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-TF-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-TF-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK-TF: [[FOR_BODY_PREHEADER]]:
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
@@ -47,10 +44,9 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
-for.body: ; preds = %for.body.preheader, %for.body
+for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
%load.a = load i8, ptr %gep.a, align 1
@@ -63,86 +59,10 @@ for.body: ; preds = %for.body.preheader,
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %exit, label %for.body
-exit: ; preds = %for.body, %entry
+exit:
ret void
}
-; Note: This test could emit a `llvm.loop.dependence.raw` mask to avoid creating
-; a dependency between the store and the load, but it is not necessary for
-; correctness.
-define i32 @alias_mask_read_after_write(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
-; CHECK-TF-LABEL: define i32 @alias_mask_read_after_write(
-; CHECK-TF-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-TF-NEXT: [[ENTRY:.*:]]
-; CHECK-TF-NEXT: [[C2:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-TF-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-TF-NEXT: [[CMP19:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-TF-NEXT: br i1 [[CMP19]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK-TF: [[FOR_BODY_PREHEADER]]:
-; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
-; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1.i64(i64 [[C2]], i64 [[B1]], i64 4)
-; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 4 x i1> [[ALIAS_MASK]] to <vscale x 4 x i32>
-; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP3]])
-; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
-; CHECK-TF-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
-; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
-; CHECK-TF-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
-; CHECK-TF-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-TF: [[VECTOR_PH]]:
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-TF: [[VECTOR_BODY]]:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP10:%.*]] = and <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], [[ALIAS_MASK]]
-; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP11]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 2 [[TMP12]], <vscale x 4 x i1> [[TMP10]])
-; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 2 [[TMP13]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
-; CHECK-TF-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_LOAD3]]
-; CHECK-TF-NEXT: [[TMP16]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
-; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-TF: [[MIDDLE_BLOCK]]:
-; CHECK-TF-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
-; CHECK-TF-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
-; CHECK-TF: [[SCALAR_PH]]:
-;
-
-
-entry:
- %cmp19 = icmp sgt i64 %n, 0
- br i1 %cmp19, label %for.body, label %exit
-
-for.body: ; preds = %entry, %for.body
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %accum = phi i32 [ 0, %entry ], [ %add2, %for.body ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %load.a = load i32, ptr %gep.a, align 2
- %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
- store i32 %load.a, ptr %gep.c, align 2
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %load.b = load i32, ptr %gep.b, align 2
- %add = add i32 %load.a, %accum
- %add2 = add i32 %add, %load.b
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %exit, label %for.body
-
-exit: ; preds = %entry, %for.body
- %result = phi i32 [ 0, %entry ], [ %add2, %for.body ]
- ret i32 %result
-}
-
define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-LABEL: define void @alias_mask_multiple(
; CHECK-TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -150,9 +70,6 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: [[A3:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-TF-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
; CHECK-TF-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-TF-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-TF-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK-TF: [[FOR_BODY_PREHEADER]]:
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
@@ -191,10 +108,9 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
-for.body: ; preds = %for.body.preheader, %for.body
+for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
%load.a = load i8, ptr %gep.a, align 1
@@ -207,7 +123,7 @@ for.body: ; preds = %for.body.preheader,
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %exit, label %for.body
-exit: ; preds = %for.body, %entry
+exit:
ret void
}
@@ -351,85 +267,3 @@ loop:
exit:
ret void
}
-
-; Test taken from: scalable-first-order-recurrence.ll. Check we don't use
-; an alias-mask with first-order recurrences, as we cannot handle the
-; splice.right with the alias-mask/clamped VF yet.
-define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
-; CHECK-TF-LABEL: define i32 @recurrence_1(
-; CHECK-TF-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-TF-NEXT: [[ENTRY:.*:]]
-; CHECK-TF-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
-; CHECK-TF-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-TF-NEXT: br label %[[FOR_PREHEADER:.*]]
-; CHECK-TF: [[FOR_PREHEADER]]:
-; CHECK-TF-NEXT: [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
-; CHECK-TF-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; CHECK-TF-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-TF-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-TF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
-; CHECK-TF: [[VECTOR_MEMCHECK]]:
-; CHECK-TF-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-TF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-TF-NEXT: [[TMP6:%.*]] = add i64 [[B1]], -4
-; CHECK-TF-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[A2]]
-; CHECK-TF-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
-; CHECK-TF-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-TF: [[VECTOR_PH]]:
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])
-; CHECK-TF-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-TF-NEXT: [[TMP14:%.*]] = mul nuw i32 [[TMP13]], 4
-; CHECK-TF-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1
-; CHECK-TF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP15]]
-; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-TF: [[VECTOR_BODY]]:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP16:%.*]] = add nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
-; CHECK-TF-NEXT: [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
-; CHECK-TF-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP18]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP20]], ptr align 4 [[TMP19]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP2]])
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
-; CHECK-TF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-TF: [[MIDDLE_BLOCK]]:
-; CHECK-TF-NEXT: [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; CHECK-TF-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP23]], i1 false)
-; CHECK-TF-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-TF-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i32> [[TMP18]], i64 [[LAST_ACTIVE_LANE]]
-; CHECK-TF-NEXT: br [[FOR_EXIT:label %.*]]
-; CHECK-TF: [[SCALAR_PH]]:
-;
-
-entry:
- br label %for.preheader
-
-for.preheader:
- %pre_load = load i32, ptr %a
- br label %scalar.body
-
-scalar.body:
- %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
- %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %arrayidx32 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.next
- %1 = load i32, ptr %arrayidx32
- %arrayidx34 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
- %add35 = add i32 %1, %0
- store i32 %add35, ptr %arrayidx34
- %lftr.wideiv = trunc i64 %indvars.iv.next to i32
- %exitcond = icmp eq i32 %lftr.wideiv, %n
- br i1 %exitcond, label %for.exit, label %scalar.body
-
-for.exit:
- ret i32 %0
-}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index 13040befa57c0..0ce087f08ef2a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -39,8 +39,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, ir<%iv>
; INITIAL-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.c>
; INITIAL-NEXT: WIDEN store vp<[[VP11]]>, ir<%add>, vp<[[VP8]]>
-; INITIAL-NEXT: CLONE ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
-; INITIAL-NEXT: CLONE ir<%exitcond.not> = icmp eq ir<%iv.next>, ir<%n>
; INITIAL-NEXT: Successor(s): vector.latch
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.latch:
@@ -86,37 +84,40 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: vector.clamped.vf.check:
; FINAL-NEXT: WIDEN-INTRINSIC vp<[[VP2:%[0-9]+]]> = call llvm.loop.dependence.war.mask(ir<%b2>, ir<%c1>, ir<1>)
; FINAL-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = num-active-lanes vp<[[VP2]]>
+; FINAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast vp<[[VP3]]>
; FINAL-NEXT: EMIT vp<%vf.is.scalar> = icmp ule vp<[[VP3]]>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = sub ir<-1>, ir<%n>
-; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP4]]>, vp<[[VP3]]>
-; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP5]]>
+; FINAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = sub ir<-1>, ir<%n>
+; FINAL-NEXT: EMIT vp<%vf.step.overflow> = icmp ult vp<[[VP5]]>, vp<[[VP3]]>
+; FINAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or vp<%vf.is.scalar>, vp<%vf.step.overflow>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP6]]>
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
; FINAL-NEXT: EMIT vp<%trip.count.minus.1> = sub ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = sub vp<[[VP3]]>, ir<1>
-; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP7]]>
+; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = sub vp<[[VP3]]>, ir<1>
+; FINAL-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, vp<[[VP8]]>
; FINAL-NEXT: EMIT vp<%n.mod.vf> = urem vp<%n.rnd.up>, vp<[[VP3]]>
; FINAL-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<%n.mod.vf>
-; FINAL-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = broadcast vp<%trip.count.minus.1>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = step-vector i64
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<[[VP9]]>, vp<[[VP8]]>
-; FINAL-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = and vp<[[VP10]]>, vp<[[VP2]]>
+; FINAL-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ vp<[[VP10]]>, vector.ph ], [ vp<%vec.ind.next>, vector.body ]
+; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ule vp<[[VP11]]>, vp<[[VP9]]>
+; FINAL-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = and vp<[[VP12]]>, vp<[[VP2]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP11]]>
+; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP13]]>
; FINAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, vp<%index>
-; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP11]]>
+; FINAL-NEXT: WIDEN ir<%ld.b> = load ir<%ptr.b>, vp<[[VP13]]>
; FINAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
-; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP11]]>
+; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP13]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<[[VP3]]>
-; FINAL-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
-; FINAL-NEXT: EMIT branch-on-cond vp<[[VP12]]>
+; FINAL-NEXT: EMIT vp<%vec.ind.next> = add vp<[[VP11]]>, vp<[[VP4]]>
+; FINAL-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; FINAL-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
index 4df5a50948c81..8696ab5dc1feb 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll
@@ -1,16 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
-; Note: First order recurrences are not supported with alias-masking.
-define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
+; Test taken from: scalable-first-order-recurrence.ll. Check we don't use
+; an alias-mask with first-order recurrences, as we cannot handle the
+; splice.right with the alias-mask/clamped VF yet.
+define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b, i32 %init_val, i32 %n) {
; CHECK-LABEL: define i32 @first_order_recurrence(
-; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], i32 [[INIT_VAL:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-NEXT: br label %[[FOR_PREHEADER:.*]]
-; CHECK: [[FOR_PREHEADER]]:
-; CHECK-NEXT: [[PRE_LOAD:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
@@ -27,14 +26,12 @@ define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b,
; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[PRE_LOAD]], i32 3
+; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[INIT_VAL]], i32 3
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
@@ -44,6 +41,7 @@ define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b,
; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[TMP8]]
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP10]], ptr align 4 [[TMP9]], <4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], splat (i64 4)
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -55,15 +53,11 @@ define i32 @first_order_recurrence(ptr nocapture readonly %a, ptr nocapture %b,
; CHECK: [[SCALAR_PH]]:
;
entry:
- br label %for.preheader
+ br label %loop
-for.preheader:
- %pre_load = load i32, ptr %a
- br label %scalar.body
-
-scalar.body:
- %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
- %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+loop:
+ %0 = phi i32 [ %init_val, %entry ], [ %1, %loop ]
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ]
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%arrayidx32 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.next
%1 = load i32, ptr %arrayidx32
@@ -72,7 +66,7 @@ scalar.body:
store i32 %add35, ptr %arrayidx34
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, %n
- br i1 %exitcond, label %for.exit, label %scalar.body
+ br i1 %exitcond, label %for.exit, label %loop
for.exit:
ret i32 %0
@@ -102,9 +96,7 @@ define void @uses_bounds_checks(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT3]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
@@ -115,6 +107,7 @@ define void @uses_bounds_checks(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP5]], ptr align 1 [[TMP6]], <4 x i1> [[TMP1]]), !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], splat (i64 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -141,3 +134,67 @@ for.body:
exit:
ret void
}
+
+; TODO: Handle the horizontal reduction outside the loop with an alias-mask.
+; We need to set values outside the alias-mask to the identity value before
+; the reduction.
+define i8 @add_reduction(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: define i8 @add_reduction(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A2:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[B1:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[A2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
+; CHECK-NEXT: [[TMP3]] = add <4 x i8> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[VEC_IND]], ptr align 1 [[TMP4]], <4 x i1> [[TMP1]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <4 x i64> [[VEC_IND3]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[TMP3]], <4 x i8> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
+; CHECK-NEXT: br [[EXIT:label %.*]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i8 [ 0, %entry ], [ %sum.1, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %sum.1 = add i8 %sum.0, %load.a
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %trunc = trunc i64 %iv to i8
+ store i8 %trunc, ptr %gep.b, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i8 %sum.1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index e9c27ce38a3db..c685f5069245f 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -13,15 +13,14 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
@@ -38,9 +37,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
@@ -52,6 +49,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP13]], ptr align 1 [[TMP14]], <4 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -59,8 +57,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[SCALAR_PH]]:
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -87,9 +84,6 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[A3:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
@@ -98,6 +92,8 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i32>
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 -1, [[N]]
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP9]], [[NUM_ACTIVE_LANES]]
@@ -114,9 +110,7 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = and <4 x i1> [[TMP12]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
@@ -127,6 +121,7 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP16]], ptr align 1 [[TMP17]], <4 x i1> [[TMP13]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -134,8 +129,7 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK: [[SCALAR_PH]]:
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -154,97 +148,20 @@ exit:
ret void
}
-; Alias masking + a simple add reduction.
-define i32 @alias_mask_with_reduction(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
-; CHECK-LABEL: define i32 @alias_mask_with_reduction(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
-; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
-; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP10]], <4 x i1> [[TMP9]], <4 x i8> poison)
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i8> poison)
-; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP12]], ptr align 1 [[TMP13]], <4 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
-; CHECK-NEXT: [[TMP15]] = add <4 x i32> [[VEC_PHI]], [[TMP14]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP15]], <4 x i32> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP17]])
-; CHECK-NEXT: br [[EXIT:label %.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %reduce = phi i32 [ 0, %entry ], [ %reduce.next, %for.body ]
- %ptr.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
- %ld.a = load i8, ptr %ptr.a, align 1
- %ptr.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
- %ld.b = load i8, ptr %ptr.b, align 1
- %add = add i8 %ld.b, %ld.a
- %ptr.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv
- store i8 %add, ptr %ptr.c, align 1
- %ext.add = zext i8 %add to i32
- %reduce.next = add nuw nsw i32 %reduce, %ext.add
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %exit, label %for.body
-
-exit:
- ret i32 %reduce.next
-}
-
define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, ptr addrspace(1) %b, ptr addrspace(1) %c, i64 %n) {
; CHECK-LABEL: define void @alias_mask_non_default_address_space(
; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr addrspace(1) [[B]] to i64
; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr addrspace(1) [[C]] to i64
-; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], [[EXIT:label %.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
@@ -261,9 +178,7 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT4]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i1> [[TMP8]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[INDEX]]
@@ -275,15 +190,15 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[C]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i8.p1(<4 x i8> [[TMP13]], ptr addrspace(1) align 1 [[TMP14]], <4 x i1> [[TMP9]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
;
entry:
- %cmp11 = icmp sgt i64 %n, 0
- br i1 %cmp11, label %for.body, label %exit
+ br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -315,6 +230,9 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP14]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 -8, [[NUM_ACTIVE_LANES]]
; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
@@ -327,10 +245,8 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IV]], splat (i64 6)
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 6)
; CHECK-NEXT: [[TMP8:%.*]] = and <4 x i1> [[TMP7]], [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP9]], <4 x i1> [[TMP8]], <4 x i8> poison)
@@ -340,8 +256,9 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP11]], ptr align 1 [[TMP12]], <4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index 979b4ff3c0e7a..1f719673eb217 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -420,11 +420,11 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; VF4-SAME: i64 [[N:%.*]], i64 [[MASK_INIT:%.*]]) {
; VF4-NEXT: [[ENTRY:.*]]:
; VF4-NEXT: [[MASK:%.*]] = and i64 [[MASK_INIT]], -4294967296
+; VF4-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; VF4-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[MASK]], 2
; VF4-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
; VF4-NEXT: [[TMP1:%.*]] = add i64 [[SMAX1]], -1
; VF4-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[MASK]]
-; VF4-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; VF4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
; VF4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; VF4: [[VECTOR_SCEVCHECK]]:
@@ -488,11 +488,11 @@ define i64 @induction_cast_chain_cleared_by_dce(i64 %n, i64 %mask.init) {
; IC2-SAME: i64 [[N:%.*]], i64 [[MASK_INIT:%.*]]) {
; IC2-NEXT: [[ENTRY:.*]]:
; IC2-NEXT: [[MASK:%.*]] = and i64 [[MASK_INIT]], -4294967296
+; IC2-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; IC2-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[MASK]], 2
; IC2-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
; IC2-NEXT: [[TMP1:%.*]] = add i64 [[SMAX1]], -1
; IC2-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[MASK]]
-; IC2-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[MASK]], 1
; IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
; IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; IC2: [[VECTOR_SCEVCHECK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
index 7a38b3ca7cc23..8e7df4d2e9a8a 100644
--- a/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/remove-redundant-trip-count-scev.ll
@@ -29,15 +29,14 @@ define void @test(ptr %base_a, ptr %base_b, i32 %ntypes) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP20:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP20]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[VEC_IND:%.*]] = add <4 x i64> [[BROADCAST_SPLAT6]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[BASE_A]], <4 x i64> [[VEC_IND1]]
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [8 x i8], ptr [[BASE_B]], i64 [[TMP20]]
; CHECK-NEXT: call void @llvm.masked.store.v4p0.p0(<4 x ptr> [[TMP3]], ptr align 8 [[TMP23]], <4 x i1> [[TMP0]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP20]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND1]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[VEC_IND_NEXT6]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From 8875c53d1f0c27f7ea2bb0c83908d0b0b529d5fe Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 12 May 2026 15:15:27 +0000
Subject: [PATCH 7/7] Fixups
---
.../Vectorize/LoopVectorizationPlanner.h | 5 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 13 +++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 ++++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
.../AArch64/alias-mask-uniforms.ll | 15 ++---
.../LoopVectorize/AArch64/alias-mask.ll | 21 +++----
.../AArch64/vplan-printing-alias-mask.ll | 23 +-------
.../VPlan/vplan-printing-alias-mask.ll | 45 +--------------
.../alias-mask-data-tail-folding-style.ll | 5 +-
.../LoopVectorize/alias-mask-needs-freeze.ll | 56 +++++++++++++++++++
.../LoopVectorize/alias-mask-small-index.ll | 47 ++++++++++++++++
.../Transforms/LoopVectorize/alias-mask.ll | 20 +++----
12 files changed, 157 insertions(+), 110 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/alias-mask-needs-freeze.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/alias-mask-small-index.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 565d05d234666..480894fc15f08 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -415,6 +415,11 @@ class VPBuilder {
return createScalarCast(CastOp, Op, ResultTy, DL);
}
+ VPValue *createScalarFreeze(VPValue *Op, Type *ResultTy, DebugLoc DL) {
+ return tryInsertInstruction(new VPInstructionWithType(
+ Instruction::Freeze, Op, ResultTy, {}, {}, DL));
+ }
+
VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy) {
return tryInsertInstruction(new VPWidenCastRecipe(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 069653dc5ba67..0eecb32d4efcd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -665,11 +665,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
Type *Ty = State.TypeAnalysis.inferScalarType(this);
Value *ZExt = Builder.CreateCast(
- Instruction::ZExt, Op,
- VectorType::get(Builder.getInt32Ty(), VecTy->getElementCount()));
- Value *Count =
+ Instruction::ZExt, Op, VectorType::get(Ty, VecTy->getElementCount()));
+ Value *NumActive =
Builder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, ZExt);
- return Builder.CreateCast(Instruction::ZExt, Count, Ty, "num.active.lanes");
+ return NumActive;
}
case VPInstruction::FirstOrderRecurrenceSplice: {
// Generate code to combine the previous and current values in vector v3.
@@ -1644,6 +1643,12 @@ void VPInstructionWithType::execute(VPTransformState &State) {
return;
}
switch (getOpcode()) {
+ case Instruction::Freeze: {
+ Value *Op = State.get(getOperand(0), VPLane(0));
+ Value *Freeze = State.Builder.CreateFreeze(Op);
+ State.set(this, Freeze, VPLane(0));
+ break;
+ }
case VPInstruction::StepVector: {
Value *StepVector =
State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 53c5d33b4071f..bf3bce50bb9cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5092,10 +5092,19 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
vputils::getOrCreateVPValueForSCEVExpr(Plan, Check.SinkStart);
Type *AddrType = TypeInfo.inferScalarType(Src);
- VPWidenIntrinsicRecipe *WARMask = new VPWidenIntrinsicRecipe(
+ // TODO: Only freeze the required pointer (not both src and sink).
+ if (Check.NeedsFreeze) {
+ Src = Builder.createScalarFreeze(Src, AddrType, DebugLoc::getUnknown());
+ Sink = Builder.createScalarFreeze(Sink, AddrType, DebugLoc::getUnknown());
+ }
+
+ // TODO: Generate loop_dependence_raw_mask when there's a read-after-write
+ // dependency between the source and the sink. This is not necessary for
+ // correctness of the mask, but using the "raw" variant prevents loads
+ // depending on the completion of stores.
+ VPWidenIntrinsicRecipe *WARMask = Builder.insert(new VPWidenIntrinsicRecipe(
Intrinsic::loop_dependence_war_mask,
- {Src, Sink, Plan.getConstantInt(AddrType, Check.AccessSize)}, I1Ty);
- Builder.insert(WARMask);
+ {Src, Sink, Plan.getConstantInt(AddrType, Check.AccessSize)}, I1Ty));
if (AliasMask)
AliasMask = Builder.createAnd(AliasMask, WARMask);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index d13a5203b3a50..dba337974e855 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -699,7 +699,7 @@ VPBlockUtils::blocksInSingleSuccessorChainBetween(VPBasicBlock *FirstBB,
}
VPValue *vputils::findIncomingAliasMask(const VPlan &Plan) {
- for (VPRecipeBase &R : *const_cast<VPlan &>(Plan).getVectorPreheader())
+ for (VPRecipeBase &R : *Plan.getVectorPreheader())
if (match(&R, m_VPInstruction<VPInstruction::IncomingAliasMask>()))
return cast<VPInstruction>(&R);
return nullptr;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
index d012e99724d01..9ec32c0d66afe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
@@ -13,9 +13,8 @@ define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n)
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
@@ -121,9 +120,8 @@ define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
@@ -189,9 +187,8 @@ define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
index 68b77544899ce..861adca3c9df0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll
@@ -10,9 +10,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
-; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
-; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
-; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-TF-NEXT: [[TMP1:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i64>
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> [[TMP1]])
; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-TF-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
@@ -75,9 +74,8 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
; CHECK-TF-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = and <vscale x 16 x i1> [[TMP2]], [[TMP5]]
-; CHECK-TF-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP7]])
-; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i64>
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> [[TMP3]])
; CHECK-TF-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-TF-NEXT: [[TMP9:%.*]] = sub i64 -1, [[N]]
; CHECK-TF-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP9]], [[NUM_ACTIVE_LANES]]
@@ -101,7 +99,7 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TF-NEXT: [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
-; CHECK-TF-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br [[EXIT_LOOPEXIT:label %.*]]
; CHECK-TF: [[SCALAR_PH]]:
@@ -138,9 +136,8 @@ define i8 @alias_masking_exit_value(ptr %ptrA, ptr %ptrB) {
; CHECK-TF-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK-TF: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-TF-NEXT: [[ALIAS_MASK:%.*]] = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 [[PTRA2]], i64 [[PTRB1]], i64 1)
-; CHECK-TF-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i32>
-; CHECK-TF-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP3]])
-; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-TF-NEXT: [[TMP1:%.*]] = zext <vscale x 16 x i1> [[ALIAS_MASK]] to <vscale x 16 x i64>
+; CHECK-TF-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> [[TMP1]])
; CHECK-TF-NEXT: [[TMP5:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i32
; CHECK-TF-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
; CHECK-TF-NEXT: [[TMP7:%.*]] = mul i8 1, [[TMP6]]
@@ -169,7 +166,7 @@ define i8 @alias_masking_exit_value(ptr %ptrA, ptr %ptrB) {
; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
; CHECK-TF-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: [[TMP19:%.*]] = xor <vscale x 16 x i1> [[TMP13]], splat (i1 true)
; CHECK-TF-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP19]], i1 false)
@@ -242,7 +239,7 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC,
; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[IV_START]])
; CHECK-TF-NEXT: [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TF-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true
-; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-TF: [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT: br [[EXIT:label %.*]]
; CHECK-TF: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 530ccf7bf8e57..377938c26906a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
; RUN: opt -passes=loop-vectorize -mattr=+sve2 -force-partial-aliasing-vectorization -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefixes=FINAL %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -46,27 +46,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
-; FINAL-NEXT: Successor(s): ir-bb<exit>
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<exit>:
-; FINAL-NEXT: No successors
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<scalar.ph>:
-; FINAL-NEXT: Successor(s): ir-bb<for.body>
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<for.body>:
-; FINAL-NEXT: IR %iv = phi i64 [ 0, %scalar.ph ], [ %iv.next, %for.body ] (extra operand: ir<0> from ir-bb<scalar.ph>)
-; FINAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
-; FINAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
-; FINAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
-; FINAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
-; FINAL-NEXT: IR %add = add i8 %ld.b, %ld.a
-; FINAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
-; FINAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
-; FINAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
-; FINAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
-; FINAL-NEXT: No successors
-; FINAL-NEXT: }
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index 0ce087f08ef2a..177c8ddb34dd9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="attachAliasMaskToHeaderMask$" -S %s 2>&1 | FileCheck --check-prefix=INITIAL %s
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -disable-output -vplan-print-after="printFinalVPlan$" -S %s 2>&1 | FileCheck --check-prefix=FINAL %s
@@ -49,28 +49,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Successor(s): middle.block
; INITIAL-EMPTY:
; INITIAL-NEXT: middle.block:
-; INITIAL-NEXT: Successor(s): ir-bb<exit>
-; INITIAL-EMPTY:
-; INITIAL-NEXT: ir-bb<exit>:
-; INITIAL-NEXT: No successors
-; INITIAL-EMPTY:
-; INITIAL-NEXT: scalar.ph:
-; INITIAL-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<0>, ir-bb<entry> ]
-; INITIAL-NEXT: Successor(s): ir-bb<for.body>
-; INITIAL-EMPTY:
-; INITIAL-NEXT: ir-bb<for.body>:
-; INITIAL-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; INITIAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
-; INITIAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
-; INITIAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
-; INITIAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
-; INITIAL-NEXT: IR %add = add i8 %ld.b, %ld.a
-; INITIAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
-; INITIAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
-; INITIAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
-; INITIAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
-; INITIAL-NEXT: No successors
-; INITIAL-NEXT: }
;
; FINAL-LABEL: VPlan for loop in 'alias_mask'
; FINAL: VPlan 'Final VPlan for VF={4},UF={1}' {
@@ -121,27 +99,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
; FINAL-NEXT: middle.block:
-; FINAL-NEXT: Successor(s): ir-bb<exit>
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<exit>:
-; FINAL-NEXT: No successors
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<scalar.ph>:
-; FINAL-NEXT: Successor(s): ir-bb<for.body>
-; FINAL-EMPTY:
-; FINAL-NEXT: ir-bb<for.body>:
-; FINAL-NEXT: IR %iv = phi i64 [ 0, %scalar.ph ], [ %iv.next, %for.body ] (extra operand: ir<0> from ir-bb<scalar.ph>)
-; FINAL-NEXT: IR %ptr.a = getelementptr inbounds i8, ptr %a, i64 %iv
-; FINAL-NEXT: IR %ld.a = load i8, ptr %ptr.a, align 1
-; FINAL-NEXT: IR %ptr.b = getelementptr inbounds i8, ptr %b, i64 %iv
-; FINAL-NEXT: IR %ld.b = load i8, ptr %ptr.b, align 1
-; FINAL-NEXT: IR %add = add i8 %ld.b, %ld.a
-; FINAL-NEXT: IR %ptr.c = getelementptr inbounds i8, ptr %c, i64 %iv
-; FINAL-NEXT: IR store i8 %add, ptr %ptr.c, align 1
-; FINAL-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
-; FINAL-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
-; FINAL-NEXT: No successors
-; FINAL-NEXT: }
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
index a3d519a7304d4..9ab6484da2b6b 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-data-tail-folding-style.ll
@@ -16,9 +16,8 @@ define void @test(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[SRC2]], i64 [[DST1]], i64 4)
-; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i32>
-; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i32
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i32 [[TMP7]], 1
; CHECK-NEXT: [[TMP8:%.*]] = sub i32 -1, [[UMAX3]]
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-needs-freeze.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-needs-freeze.ll
new file mode 100644
index 0000000000000..9463f5aee0601
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-needs-freeze.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^vector.ph:" --version 5
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+
+; Tests the inputs to the loop dependence masks have freezes for forked pointers.
+define void @alias_mask_forked_pointer_needs_freeze(ptr %base1, ptr %base2,
+; CHECK-LABEL: define void @alias_mask_forked_pointer_needs_freeze(
+; CHECK-SAME: ptr [[BASE1:%.*]], ptr [[BASE2:%.*]], ptr [[DEST:%.*]], ptr [[PREDICATES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PREDICATES4:%.*]] = ptrtoaddr ptr [[PREDICATES]] to i64
+; CHECK-NEXT: [[BASE23:%.*]] = ptrtoaddr ptr [[BASE2]] to i64
+; CHECK-NEXT: [[BASE12:%.*]] = ptrtoaddr ptr [[BASE1]] to i64
+; CHECK-NEXT: [[DEST1:%.*]] = ptrtoaddr ptr [[DEST]] to i64
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[PREDICATES4]], i64 [[DEST1]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[BASE23]]
+; CHECK-NEXT: [[TMP2:%.*]] = freeze i64 [[DEST1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[TMP1]], i64 [[TMP2]], i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and <4 x i1> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze i64 [[BASE12]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze i64 [[DEST1]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[TMP5]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = and <4 x i1> [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext <4 x i1> [[TMP8]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP9]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP11]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP12]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+;
+ ptr %dest, ptr %predicates,
+ i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p.gep = getelementptr inbounds i32, ptr %predicates, i64 %iv
+ %p = load i32, ptr %p.gep, align 4
+ %cmp = icmp eq i32 %p, 0
+ %src = select i1 %cmp, ptr %base2, ptr %base1
+ %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
+ %src.val = load float, ptr %src.gep, align 4
+ %dst.gep = getelementptr inbounds float, ptr %dest, i64 %iv
+ store float %src.val, ptr %dst.gep, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask-small-index.ll b/llvm/test/Transforms/LoopVectorize/alias-mask-small-index.ll
new file mode 100644
index 0000000000000..444221d29b47c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask-small-index.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^vector.ph:" --version 5
+; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s
+
+target datalayout = "p0:64:64:64:16"
+
+; Tests we correctly handle the extend for "NUM_ACTIVE_LANES" when the GEP index
+; is smaller than the IV type.
+define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-LABEL: define void @alias_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[B2:%.*]] = ptrtoaddr ptr [[B]] to i16
+; CHECK-NEXT: [[C1:%.*]] = ptrtoaddr ptr [[C]] to i16
+; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
+; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i16(i16 [[B2]], i16 [[C1]], i16 1)
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 -1, [[N]]
+; CHECK-NEXT: [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP6]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %add = add i8 %load.b, %load.a
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 %add, ptr %gep.c, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index c685f5069245f..1ec0cb4dd15d4 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -16,9 +16,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
@@ -89,9 +88,8 @@ define void @alias_mask_multiple(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[A3]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i1> [[TMP2]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i32>
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
@@ -157,9 +155,8 @@ define void @alias_mask_non_default_address_space(ptr addrspace(1) noalias %a, p
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
@@ -227,9 +224,8 @@ define void @alias_mask_known_trip_count(ptr noalias %a, ptr %b, ptr %c) {
; CHECK-NEXT: br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK: [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[B2]], i64 [[C1]], i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
+; CHECK-NEXT: [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[NUM_ACTIVE_LANES]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP14]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
More information about the llvm-commits
mailing list