[llvm] [LV] Use wide lane masks as the canonical form when tail-folding & interleaving (PR #209484)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 05:19:59 PDT 2026
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/209484
>From 337b77e9f0ce737ca55b6f99e6b25591ca10bfec Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 8 Jul 2026 14:00:32 +0000
Subject: [PATCH 01/14] [LV] Use wide lane masks as the canonical form when
tail-folding & interleaving
Previously, when passing -enable-wide-lane-mask to request the use of wide
lane masks, tryToReplaceALMWithWideALM would attempt to replace several
ActiveLaneMasks created during unrolling with a single mask + extracts.
This PR removes tryToReplaceALMWithWideALM from VPTransforms and makes
wide lane masks the canonical form when interleaving a tail-folded loop
which uses ActiveLaneMask for control flow.
During unrolling, the multiplier is set to UF for ActiveLaneMask and one
ExtractSubvectorForPart instruction for each part is created. Unnecessary
ExtractSubvectorForParts are removed during the generate stage if the loop
is not unrolled, otherwise this creates an extract subvector for each part.
This does not change the default behaviour for tail-folded loops, which
are still not considered for interleaving by default. The decision on
whether to use a single wide active lane mask with extracts or to split
it up into multiple smaller masks is now left to the backend.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 6 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 38 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 111 +------
.../Transforms/Vectorize/VPlanTransforms.h | 1 -
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 25 ++
.../AArch64/fixed-wide-lane-mask.ll | 38 +--
.../AArch64/masked-call-scalarize.ll | 15 +-
.../LoopVectorize/AArch64/masked-call.ll | 79 ++---
.../LoopVectorize/AArch64/pr73894.ll | 8 +-
.../AArch64/sve-tail-folding-unroll.ll | 50 ++-
.../AArch64/sve-wide-lane-mask.ll | 303 +++++++-----------
.../AArch64/uniform-args-call-variants.ll | 134 ++++----
.../LoopVectorize/ARM/active-lane-mask.ll | 2 +-
15 files changed, 367 insertions(+), 476 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d892ff2a0fa2b..e902dcbb11ceb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -255,11 +255,6 @@ static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"Use predicated EVL instructions for tail folding. If EVL "
"is unsupported, fallback to data-without-lane-mask.")));
-cl::opt<bool> llvm::EnableWideActiveLaneMask(
- "enable-wide-lane-mask", cl::init(false), cl::Hidden,
- cl::desc("Enable use of wide lane masks when used for control flow in "
- "tail-folded loops"));
-
static cl::opt<bool> EnableInterleavedMemAccesses(
"enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
@@ -1219,15 +1214,6 @@ class LoopVectorizationCostModel {
return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
}
- /// Returns true if the use of wide lane masks is requested and the loop is
- /// using tail-folding with a lane mask for control flow.
- bool useWideActiveLaneMask() const {
- if (!EnableWideActiveLaneMask)
- return false;
-
- return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
- }
-
/// Returns true if the instructions in this block requires predication
/// for any reason, e.g. because tail folding now requires a predicate
/// or because the block in the original loop was predicated.
@@ -3651,12 +3637,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// 3. We don't interleave if we think that we will spill registers to memory
// due to the increased register pressure.
- // Only interleave tail-folded loops if wide lane masks are requested, as the
- // overhead of multiple instructions to calculate the predicate is likely
- // not beneficial. If an epilogue is not allowed for any other reason,
- // do not interleave.
- if (!CM.isEpilogueAllowed() &&
- !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()))
+ // Do not interleave tail-folded loops, as the overhead of multiple
+ // instructions to calculate the predicate is likely not beneficial.
+ // If an epilogue is not allowed for any other reason, do not interleave.
+ if (!CM.isEpilogueAllowed() || CM.preferTailFoldedLoop())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..b0bc95d7adcc9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1235,15 +1235,18 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
enum {
- FirstOrderRecurrenceSplice =
- Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
- // values of a first-order recurrence.
+ FirstOrderRecurrenceSplice = Instruction::OtherOpsEnd +
+ 1, // Combines the incoming and previous
+ // values of a first-order recurrence.
Not,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
+ // Extracts each unrolled part of a widened active lane mask when used
+ // for control flow.
+ ExtractSubvectorForPart,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index d575d1f150e7a..842d2697148c6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -447,6 +447,12 @@ m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
return m_ExtractLastLane(m_ExtractLastPart(Op0));
}
+template <typename Op0_t, typename Op1_t>
+inline VPInstruction_match<VPInstruction::ExtractSubvectorForPart, Op0_t, Op1_t>
+m_ExtractSubvectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(Op0, Op1);
+}
+
template <typename Op0_t>
inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
m_ExtractPenultimateElement(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 50de4c5530f02..c99e78f24082b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -666,6 +666,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::ResumeForEpilogue:
+ case VPInstruction::ExtractSubvectorForPart:
return 2;
case Instruction::InsertElement:
case Instruction::Select:
@@ -801,12 +802,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
- if (State.VF.isScalar())
+ unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ if (State.VF.isScalar() && Multiplier == 1)
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC,
Name);
- ElementCount EC = State.VF.multiplyCoefficientBy(
- cast<VPConstantInt>(getOperand(2))->getZExtValue());
+ ElementCount EC = State.VF.multiplyCoefficientBy(Multiplier);
auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC);
return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
{PredTy, ScalarTC->getType()},
@@ -1112,6 +1113,33 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
+ case VPInstruction::ExtractSubvectorForPart: {
+ Type *DstTy = VectorType::get(getScalarType(), State.VF);
+ unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
+
+ // Return Src if it's an ActiveLaneMask with a multiplier of 1.
+ Value *Src = State.get(getOperand(0));
+ if (Part == 0 && match(getOperand(0),
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_One())))
+ return Src;
+
+ // If the VF is scalar & this is an extract of an active lane mask,
+ // generate an ICMP directly.
+ VPValue *Start, *TC;
+ if (State.VF.isScalar() &&
+ match(getOperand(0),
+ m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
+ Value *StartV = State.get(Start);
+ if (Part > 0)
+ StartV = Builder.CreateAdd(State.get(Start), State.get(getOperand(1)));
+ return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
+ State.get(TC));
+ }
+
+ auto *Idx = ConstantInt::get(Builder.getInt64Ty(),
+ State.VF.getKnownMinValue() * Part);
+ return Builder.CreateExtractVector(DstTy, Src, Idx);
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1637,6 +1665,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLastActive:
+ case VPInstruction::ExtractSubvectorForPart:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -1810,6 +1839,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
+ case VPInstruction::ExtractSubvectorForPart:
+ O << "extract-subvector-for-part";
+ break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5f125aecb005a..9c5b8323b786b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2037,105 +2037,6 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
-/// Try to replace multiple active lane masks used for control flow with
-/// a single, wide active lane mask instruction followed by multiple
-/// extract subvector intrinsics. This applies to the active lane mask
-/// instructions both in the loop and in the preheader.
-/// Incoming values of all ActiveLaneMaskPHIs are updated to use the
-/// new extracts from the first active lane mask, which has it's last
-/// operand (multiplier) set to UF.
-static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
- unsigned UF) {
- if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1)
- return false;
-
- VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
- auto *Term = &ExitingVPBB->back();
-
- using namespace llvm::VPlanPatternMatch;
- if (!match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue())))))
- return false;
-
- auto *Header = cast<VPBasicBlock>(VectorRegion->getEntry());
- LLVMContext &Ctx = Plan.getContext();
-
- auto ExtractFromALM = [&](VPInstruction *ALM,
- SmallVectorImpl<VPValue *> &Extracts) {
- DebugLoc DL = ALM->getDebugLoc();
- for (unsigned Part = 0; Part < UF; ++Part) {
- SmallVector<VPValue *> Ops;
- Ops.append({ALM, Plan.getConstantInt(64, VF.getKnownMinValue() * Part)});
- auto *Ext =
- new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops,
- IntegerType::getInt1Ty(Ctx), {}, {}, DL);
- Extracts[Part] = Ext;
- Ext->insertAfter(ALM);
- }
- };
-
- // Create a list of each active lane mask phi, ordered by unroll part.
- SmallVector<VPActiveLaneMaskPHIRecipe *> Phis(UF, nullptr);
- for (VPRecipeBase &R : Header->phis()) {
- auto *Phi = dyn_cast<VPActiveLaneMaskPHIRecipe>(&R);
- if (!Phi)
- continue;
- VPValue *Index = nullptr;
- match(Phi->getBackedgeValue(),
- m_ActiveLaneMask(m_VPValue(Index), m_VPValue(), m_VPValue()));
- assert(Index && "Expected index from ActiveLaneMask instruction");
-
- uint64_t Part;
- if (match(Index,
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
- m_VPValue(), m_Mul(m_VPValue(), m_ConstantInt(Part)))))
- Phis[Part] = Phi;
- else {
- // Anything other than a CanonicalIVIncrementForPart is part 0
- assert(!match(
- Index,
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()));
- Phis[0] = Phi;
- }
- }
-
- assert(all_of(Phis, not_equal_to(nullptr)) &&
- "Expected one VPActiveLaneMaskPHIRecipe for each unroll part");
-
- auto *EntryALM = cast<VPInstruction>(Phis[0]->getStartValue());
- auto *LoopALM = cast<VPInstruction>(Phis[0]->getBackedgeValue());
-
- assert((EntryALM->getOpcode() == VPInstruction::ActiveLaneMask &&
- LoopALM->getOpcode() == VPInstruction::ActiveLaneMask) &&
- "Expected incoming values of Phi to be ActiveLaneMasks");
-
- // When using wide lane masks, the return type of the get.active.lane.mask
- // intrinsic is VF x UF (last operand).
- VPValue *ALMMultiplier = Plan.getConstantInt(64, UF);
- EntryALM->setOperand(2, ALMMultiplier);
- LoopALM->setOperand(2, ALMMultiplier);
-
- // Create UF x extract vectors and insert into preheader.
- SmallVector<VPValue *> EntryExtracts(UF);
- ExtractFromALM(EntryALM, EntryExtracts);
-
- // Create UF x extract vectors and insert before the loop compare & branch,
- // updating the compare to use the first extract.
- SmallVector<VPValue *> LoopExtracts(UF);
- ExtractFromALM(LoopALM, LoopExtracts);
- VPInstruction *Not = cast<VPInstruction>(Term->getOperand(0));
- Not->setOperand(0, LoopExtracts[0]);
-
- // Update the incoming values of active lane mask phis.
- for (unsigned Part = 0; Part < UF; ++Part) {
- Phis[Part]->setStartValue(EntryExtracts[Part]);
- Phis[Part]->setBackedgeValue(LoopExtracts[Part]);
- }
-
- return true;
-}
-
/// Try to simplify the branch condition of \p Plan. This may restrict the
/// resulting plan to \p BestVF and \p BestUF.
static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
@@ -2152,9 +2053,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
m_CombineOr(m_CanIVInc, m_c_Add(m_CanIVInc, m_LiveIn())),
m_VPValue())) ||
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue()))))) {
+ m_VPValue(), m_VPValue(), m_VPValue())))) ||
+ match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount or BranchOnCond(Not(ActiveLaneMask)).
+ // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask))
+ // or BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMask), 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
@@ -2198,8 +2103,8 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF);
- MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
+ bool MadeChange =
+ simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
if (MadeChange) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..1c9959665ee7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -39,7 +39,6 @@ class VPRecipeBuilder;
struct VFRange;
LLVM_ABI_FOR_TEST extern cl::opt<bool> VerifyEachVPlan;
-LLVM_ABI_FOR_TEST extern cl::opt<bool> EnableWideActiveLaneMask;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_ABI_FOR_TEST extern cl::opt<bool> VPlanPrintBeforeAll;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a430de94cca14..4c15afc0466e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -471,6 +471,31 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
+ if (match(&R,
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(Op1), m_VPValue(Op2)))) {
+ auto *ALM = cast<VPInstruction>(&R);
+ auto *User = ALM->getSingleUser();
+ // Widen ActiveLaneMask when used for control flow.
+ if (User && match(User, m_ExtractSubvectorForPart(m_Specific(ALM),
+ m_ZeroInt()))) {
+ addUniformForAllParts(ALM);
+ ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ continue;
+ }
+ }
+
+ if (match(&R, m_ExtractSubvectorForPart(m_VPValue(Op0), m_VPValue(Op1)))) {
+ VPRecipeBase *InsertPt = &R;
+ for (unsigned Part = 1; Part != UF; ++Part) {
+ auto *Ext = new VPInstruction(VPInstruction::ExtractSubvectorForPart,
+ {Op0, Plan.getConstantInt(64, Part)});
+ Ext->insertAfter(InsertPt);
+ addRecipeForPart(&R, Ext, Part);
+ InsertPt = Ext;
+ }
+ continue;
+ }
+
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {
addUniformForAllParts(SingleDef);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index 17cc76c52c7e3..bff2a61711124 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
target triple = "aarch64-unknown-linux"
@@ -10,13 +10,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask(
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: br label [[ENTRY:%.*]]
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
-; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[TMP3:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -32,20 +32,20 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: br label [[ENTRY:%.*]]
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
-; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
; CHECK-UF4-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: br label [[VECTOR_BODY1:%.*]]
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
-; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[ENTRY]] ], [ [[TMP9:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[ENTRY]] ], [ [[TMP10:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[ENTRY]] ], [ [[TMP12:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-UF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-UF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -59,13 +59,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP19]], <4 x i1> [[ACTIVE_LANE_MASK6]])
; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
-; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
-; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
; CHECK-UF4-NEXT: [[TMP9]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
; CHECK-UF4-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
; CHECK-UF4-NEXT: [[TMP20:%.*]] = xor i1 [[TMP21]], true
-; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF4: middle.block:
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index 1ae5f9e065c26..8aa223c0f927c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -101,13 +101,14 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT9:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[PRED_STORE_CONTINUE8]] ]
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
@@ -144,9 +145,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE8]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE8]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT9]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP13]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index cb2eefcbc9b91..631fed7ef89f2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -82,13 +82,14 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -100,9 +101,9 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -215,13 +216,14 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -239,9 +241,9 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP13]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -373,13 +375,14 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP19]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP23]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -403,9 +406,9 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP22]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -680,13 +683,14 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -698,9 +702,9 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -855,15 +859,16 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP21]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
@@ -884,9 +889,9 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> splat (double -0.000000e+00)
; TFA_INTERLEAVE-NEXT: [[TMP16]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP14]], <vscale x 2 x double> [[TMP15]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP20]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -991,6 +996,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1013,8 +1019,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index 3e61a933d0ecb..a01923d1ce361 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -12,6 +12,7 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -30,21 +31,22 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK: pred.load.continue:
; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
; CHECK-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5]]
-; CHECK: pred.load.if4:
+; CHECK: pred.load.if3:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]]
-; CHECK: pred.load.continue5:
+; CHECK: pred.load.continue4:
; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF4]] ]
; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP8]], i32 [[VEC_PHI]])
; CHECK-NEXT: [[TMP16:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI3]])
; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]]
; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
; CHECK-NEXT: [[TMP20:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 83d9a717d04d2..55b7b14c6139e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -13,12 +13,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 2
; CHECK-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 12)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -29,6 +28,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
+; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]]
; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]]
@@ -37,13 +38,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP57]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK8]])
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP60]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK9]])
; CHECK-NEXT: [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]]
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP1]]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP28]]
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP30]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP9]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP10]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP11]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 12)
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP36:%.*]] = xor i1 [[TMP35]], true
; CHECK-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -76,12 +75,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP5]], 2
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 12)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -92,6 +90,8 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[COND_PTR:%.*]], i64 [[INDEX6]]
+; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]]
; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]]
@@ -116,13 +116,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP79]], <vscale x 4 x i1> [[TMP71]])
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP82]], <vscale x 4 x i1> [[TMP72]])
; CHECK-NEXT: [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP6]]
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP1]]
-; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP28]]
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP30]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP21]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP22]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP23]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 12)
; CHECK-NEXT: [[TMP66:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP67:%.*]] = xor i1 [[TMP66]], true
; CHECK-NEXT: br i1 [[TMP67]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 2e6e3a41addc5..f77faf7eac54e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
-; RUN: opt -S --passes=loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix CHECK-TF
-; RUN: opt -S --passes=forceattrs,loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
+; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
target triple = "aarch64-unknown-linux"
@@ -10,115 +9,80 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-UF1-LABEL: define void @scalable_wide_active_lane_mask(
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: br label [[VECTOR_PH1:%.*]]
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF1-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
-; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH1]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-UF1-NEXT: [[TMP6:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-UF1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP6]], ptr align 1 [[TMP13]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP18]]
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP3]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP4]], ptr align 1 [[TMP5]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP14:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
-; CHECK-UF1-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF1: middle.block:
;
; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: br label [[VECTOR_PH1:%.*]]
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
-; CHECK-UF4-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 4
-; CHECK-UF4-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
-; CHECK-UF4-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
-; CHECK-UF4-NEXT: [[TMP17:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-UF4-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
-; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP16]], [[VECTOR_PH1]] ], [ [[TMP55:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 16 x i1> [ [[TMP17]], [[VECTOR_PH1]] ], [ [[TMP56:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 16 x i1> [ [[TMP18]], [[VECTOR_PH1]] ], [ [[TMP57:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[VECTOR_PH1]] ], [ [[TMP58:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP32:%.*]] = shl nuw nsw i64 [[TMP1]], 1
-; CHECK-UF4-NEXT: [[TMP29:%.*]] = mul nuw nsw i64 [[TMP1]], 3
-; CHECK-UF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP32]]
-; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP29]]
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP20]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP24]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP33]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP30]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[TMP25:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP26:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD9]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP27:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD10]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP28:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD11]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP32]]
-; CHECK-UF4-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP29]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP25]], ptr align 1 [[TMP35]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP26]], ptr align 1 [[TMP39]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK6]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP27]], ptr align 1 [[TMP42]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK7]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP28]], ptr align 1 [[TMP45]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK8]])
-; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP62]]
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP17]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP15]], ptr align 1 [[TMP19]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP16]], ptr align 1 [[TMP20]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP58]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 48)
-; CHECK-UF4-NEXT: [[TMP57]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 32)
-; CHECK-UF4-NEXT: [[TMP56]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
-; CHECK-UF4-NEXT: [[TMP55]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-UF4-NEXT: [[TMP59:%.*]] = extractelement <vscale x 16 x i1> [[TMP55]], i64 0
-; CHECK-UF4-NEXT: [[TMP60:%.*]] = xor i1 [[TMP59]], true
-; CHECK-UF4-NEXT: br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4-NEXT: [[TMP21]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP22]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
+; CHECK-UF4-NEXT: [[TMP23]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 32)
+; CHECK-UF4-NEXT: [[TMP24]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 48)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP21]], i64 0
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true
+; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF4: middle.block:
;
-; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask(
-; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-TF-NEXT: entry:
-; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-TF: vector.ph:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-TF: vector.body:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP10]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 16 x i8> poison)
-; CHECK-TF-NEXT: [[TMP13:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-TF-NEXT: [[TMP14:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]], splat (i8 3)
-; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 [[TMP10]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]])
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP19]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
-; CHECK-TF-NEXT: [[TMP20]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 16 x i1> [[TMP20]], i64 0
-; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
-; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-TF: middle.block:
-;
entry:
br label %for.body
@@ -142,123 +106,85 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF1-NEXT: entry:
; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
; CHECK-UF1: for.body.preheader:
; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF1-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-UF1-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP3]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP4]], ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP7:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP7]], true
-; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF1: middle.block:
;
; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF4-NEXT: entry:
; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
; CHECK-UF4: for.body.preheader:
; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
-; CHECK-UF4-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP14:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
-; CHECK-UF4-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
-; CHECK-UF4-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-UF4-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP11]], [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[TMP12]], [[VECTOR_PH]] ], [ [[TMP51:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], [[VECTOR_PH]] ], [ [[TMP52:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 2 x i1> [ [[TMP14]], [[VECTOR_PH]] ], [ [[TMP53:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP21:%.*]] = shl nuw nsw i64 [[TMP1]], 1
-; CHECK-UF4-NEXT: [[TMP24:%.*]] = mul nuw nsw i64 [[TMP1]], 3
-; CHECK-UF4-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP21]]
-; CHECK-UF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP24]]
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP15]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP29]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP22]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP25]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP17:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD9]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP18:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD10]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP19:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD11]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP34:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP37:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP21]]
-; CHECK-UF4-NEXT: [[TMP40:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP24]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP30]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP17]], ptr align 8 [[TMP34]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP18]], ptr align 8 [[TMP37]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK7]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP19]], ptr align 8 [[TMP40]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK8]])
-; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD4]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD5]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP20]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP53]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
-; CHECK-UF4-NEXT: [[TMP52]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
-; CHECK-UF4-NEXT: [[TMP51]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; CHECK-UF4-NEXT: [[TMP50]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-UF4-NEXT: [[TMP54:%.*]] = extractelement <vscale x 2 x i1> [[TMP50]], i64 0
-; CHECK-UF4-NEXT: [[TMP55:%.*]] = xor i1 [[TMP54]], true
-; CHECK-UF4-NEXT: br i1 [[TMP55]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF4-NEXT: [[TMP21]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP22]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-UF4-NEXT: [[TMP23]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP24]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP21]], i64 0
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true
+; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF4: middle.block:
;
-; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask_double(
-; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-TF-NEXT: entry:
-; CHECK-TF-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-TF-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-TF: for.body.preheader:
-; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-TF: vector.ph:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-TF: vector.body:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP10]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; CHECK-TF-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-TF-NEXT: [[TMP14:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD2]], splat (double 3.000000e+00)
-; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP10]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP13]], ptr align 8 [[TMP15]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP19]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; CHECK-TF-NEXT: [[TMP20]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 2 x i1> [[TMP20]], i64 0
-; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
-; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-TF: middle.block:
-;
entry:
%cmp6 = icmp sgt i64 %n, 0
br i1 %cmp6, label %for.body, label %for.end
@@ -279,4 +205,3 @@ for.end:
}
attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
-
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index 094ce721f20c6..0a5a00af05e9d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -42,30 +42,31 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -125,30 +126,31 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -196,38 +198,40 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: entry:
; INTERLEAVE-NEXT: br label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[N]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[N]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]]
+; INTERLEAVE-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]]
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
-; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[PRED_STORE_CONTINUE4]] ]
+; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE3]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi i1 [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE3]] ]
; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; INTERLEAVE: pred.store.if:
-; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8
-; INTERLEAVE-NEXT: [[TMP2:%.*]] = call double @foo(double [[TMP1]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8
+; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call double @foo(double [[TMP3]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]]
; INTERLEAVE: pred.store.continue:
-; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; INTERLEAVE: pred.store.if3:
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP4]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = load double, ptr [[TMP5]], align 8
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call double @foo(double [[TMP6]], i64 [[TMP4]]) #[[ATTR4]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP4]]
-; INTERLEAVE-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
-; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; INTERLEAVE: pred.store.continue4:
+; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; INTERLEAVE: pred.store.if2:
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP6]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP7]], align 8
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = call double @foo(double [[TMP8]], i64 [[TMP6]]) #[[ATTR4]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP6]]
+; INTERLEAVE-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8
+; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE3]]
+; INTERLEAVE: pred.store.continue3:
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP9]], [[N]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
-; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i64 [[INDEX_NEXT]], 1
+; INTERLEAVE-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[N]]
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP11]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
index fb205a2927cd4..3c28a102dd89b 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -S | FileCheck %s
target triple = "thumbv8.1m.main-arm-unknown-eabihf"
>From 0733140465606761ae1308d06f63d846bc301059 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 14 Jul 2026 14:23:08 +0000
Subject: [PATCH 02/14] - Simplify ExtractSubvectorForPart recipe for removing
unnecessary extract - Move ExtractSubvectorForPart unrolling to
unrollRecipeByUF - Fix test failures
---
llvm/lib/Transforms/Vectorize/VPlan.h | 6 +++---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +++-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 18 ++++++------------
.../VPlan/AArch64/sve-tail-folding-forced.ll | 6 ++++--
.../VPlan/AArch64/vplan-printing-alias-mask.ll | 1 +
.../VPlan/AArch64/vplan-printing.ll | 6 ++++--
6 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b0bc95d7adcc9..f92a43768d3fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1235,9 +1235,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
enum {
- FirstOrderRecurrenceSplice = Instruction::OtherOpsEnd +
- 1, // Combines the incoming and previous
- // values of a first-order recurrence.
+ FirstOrderRecurrenceSplice =
+ Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
+ // values of a first-order recurrence.
Not,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c99e78f24082b..f353444ebd334 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1114,13 +1114,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
case VPInstruction::ExtractSubvectorForPart: {
+ Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
- // Return Src if it's an ActiveLaneMask with a multiplier of 1.
- Value *Src = State.get(getOperand(0));
- if (Part == 0 && match(getOperand(0),
- m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_One())))
+ if (Src->getType() == DstTy)
return Src;
// If the VF is scalar & this is an extract of an active lane mask,
@@ -1131,7 +1129,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
- StartV = Builder.CreateAdd(State.get(Start), State.get(getOperand(1)));
+ StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
State.get(TC));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 4c15afc0466e2..2e08a5d88a2b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -332,6 +332,12 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
+ if (match(&R, m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(
+ m_VPValue(Op), m_VPValue()))) {
+ Copy->setOperand(0, Op);
+ Copy->setOperand(1, Plan.getConstantInt(64, Part));
+ continue;
+ }
if (isa<VPVectorPointerRecipe, VPWidenCanonicalIVRecipe>(R)) {
VPBuilder Builder(&R);
const DataLayout &DL = Plan.getDataLayout();
@@ -484,18 +490,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
}
}
- if (match(&R, m_ExtractSubvectorForPart(m_VPValue(Op0), m_VPValue(Op1)))) {
- VPRecipeBase *InsertPt = &R;
- for (unsigned Part = 1; Part != UF; ++Part) {
- auto *Ext = new VPInstruction(VPInstruction::ExtractSubvectorForPart,
- {Op0, Plan.getConstantInt(64, Part)});
- Ext->insertAfter(InsertPt);
- addRecipeForPart(&R, Ext, Part);
- InsertPt = Ext;
- }
- continue;
- }
-
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {
addUniformForAllParts(SingleDef);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 15e57eda91e0e..0b3851a07919b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -20,20 +20,22 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
; VPLANS-NEXT: vp<[[INDV:%[0-9]+]]> = CANONICAL-IV
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.body:
-; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%.+]]>
+; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY_EXTRACT]]>, vp<[[LANEMASK_LOOP:%.+]]>
; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>, vp<[[VF]]>
; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer i32, ir<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
; VPLANS-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index c257a5a14cc27..a208e8fe477a0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -26,6 +26,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 91cb168e423ef..709bb62fd9600 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -138,13 +138,14 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vp<[[CAN_IV:%.+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%[0-9]+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
@@ -157,7 +158,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
>From 536da07d5a12c0b08e50281251aeecb7a9ef3662 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 10:01:55 +0000
Subject: [PATCH 03/14] - Rebase & update vplan-printing-alias-mask.ll
---
.../LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index a208e8fe477a0..97868414a8d70 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -31,7 +31,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = and vp<[[VP8]]>, vp<[[VP4]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP9]]>
@@ -42,7 +42,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
>From 23b783fcb437a2153010409c4107214ad69ede76 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 17 Jul 2026 13:16:29 +0000
Subject: [PATCH 04/14] - Add ActiveLaneMaskForControlFlow - Remove Op1/Op2
---
llvm/lib/Transforms/Vectorize/VPlan.h | 1 +
.../lib/Transforms/Vectorize/VPlanPatternMatch.h | 9 +++++++++
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 +++++++++++++---
.../lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 15 +++++----------
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 8 ++++++--
.../VPlan/AArch64/sve-tail-folding-forced.ll | 4 ++--
.../VPlan/AArch64/vplan-printing-alias-mask.ll | 4 ++--
.../VPlan/AArch64/vplan-printing.ll | 4 ++--
9 files changed, 45 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f92a43768d3fc..8eb45eb3c3d00 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1244,6 +1244,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
+ ActiveLaneMaskForControlFlow,
// Extracts each unrolled part of a widened active lane mask when used
// for control flow.
ExtractSubvectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 842d2697148c6..f4858f0d26f7a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -465,6 +465,15 @@ m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
}
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPInstruction_match<VPInstruction::ActiveLaneMaskForControlFlow, Op0_t,
+ Op1_t, Op2_t>
+m_ActiveLaneMaskForControlFlow(const Op0_t &Op0, const Op1_t &Op1,
+ const Op2_t &Op2) {
+ return m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>(Op0, Op1,
+ Op2);
+}
+
inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
return m_VPInstruction<VPInstruction::AnyOf>();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f353444ebd334..a221c1f90f423 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -505,6 +505,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
assert(Op0Ty->isIntegerTy() && "expected integer operand");
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
@@ -671,6 +672,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::InsertElement:
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::ReductionStartVector:
return 3;
case Instruction::Call:
@@ -794,7 +796,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
Name);
}
- case VPInstruction::ActiveLaneMask: {
+ case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow: {
// Get first lane of vector induction variable.
Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
// Get the original loop tripcount.
@@ -1126,7 +1129,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
VPValue *Start, *TC;
if (State.VF.isScalar() &&
match(getOperand(0),
- m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
+ m_ActiveLaneMaskForControlFlow(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
@@ -1445,7 +1449,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
TargetTransformInfo::SK_Splice, cast<VectorType>(VectorTy),
cast<VectorType>(VectorTy), {}, Ctx.CostKind, -1);
}
- case VPInstruction::ActiveLaneMask: {
+ case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow: {
Type *ArgTy = getOperand(0)->getScalarType();
unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
@@ -1657,6 +1662,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::IncomingAliasMask:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
@@ -1714,6 +1720,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return vputils::onlyFirstLaneUsed(this);
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -1786,6 +1793,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ActiveLaneMask:
O << "active lane mask";
break;
+ case VPInstruction::ActiveLaneMaskForControlFlow:
+ O << "active lane mask control flow";
+ break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9c5b8323b786b..205d49c5450a3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2055,11 +2055,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
m_VPValue(), m_VPValue(), m_VPValue())))) ||
match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
- m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue()),
m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask))
- // or BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMask), 0))
+ // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
+ // BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMaskForControlFlow),
+ // 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 2e08a5d88a2b2..45fdc1180ac50 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -477,17 +477,12 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
- if (match(&R,
- m_ActiveLaneMask(m_VPValue(), m_VPValue(Op1), m_VPValue(Op2)))) {
+ if (match(&R, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
- auto *User = ALM->getSingleUser();
- // Widen ActiveLaneMask when used for control flow.
- if (User && match(User, m_ExtractSubvectorForPart(m_Specific(ALM),
- m_ZeroInt()))) {
- addUniformForAllParts(ALM);
- ALM->setOperand(2, Plan.getConstantInt(64, UF));
- continue;
- }
+ addUniformForAllParts(ALM);
+ ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ continue;
}
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index ecf5fea92f4df..5f124bb6e7ccd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -196,10 +196,14 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
// loop region's abstract header mask; afterwards it is an active lane mask
// (an intrinsic or a phi), or the icmp checked below.
if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
- match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()))
+ match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()) ||
+ match(Mask,
+ m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>()))
continue;
- if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
+ if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
+ match(Mask, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue())))
continue;
CmpPredicate Pred;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 0b3851a07919b..85e15c78c0ce0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -19,7 +19,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask control flow vp<[[VF_PER_PART]]>, vp<[[TC]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
@@ -34,7 +34,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask control flow vp<[[INC]]>, vp<[[TC]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 97868414a8d70..3226ed2ba779c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -25,7 +25,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
-; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow ir<0>, ir<%n>, ir<1>
; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
@@ -41,7 +41,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<%index.next>, ir<%n>, ir<1>
; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 709bb62fd9600..de30874ee933d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -137,7 +137,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow vp<%index.part.next>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -157,7 +157,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<[[PART_IDX]]>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
>From a588ea0b44c1312a7ad1df45623d8234c86fe3d3 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 20 Jul 2026 15:59:00 +0000
Subject: [PATCH 05/14] - Rename ActiveLaneMaskForControlFlow ->
WideActiveLaneMask - Rename ExtractSubvectorForPart -> ExtractVectorForPart -
Update comments in VPTransforms to reflect wide ALM + extracts
---
llvm/lib/Transforms/Vectorize/VPlan.h | 7 ++--
.../Transforms/Vectorize/VPlanLowering.cpp | 36 ++++++++++++-------
.../Transforms/Vectorize/VPlanPatternMatch.h | 16 ++++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 30 ++++++++--------
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 +++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 6 ++--
.../Transforms/Vectorize/VPlanVerifier.cpp | 7 ++--
.../VPlan/AArch64/sve-tail-folding-forced.ll | 8 ++---
.../AArch64/vplan-printing-alias-mask.ll | 8 ++---
.../VPlan/AArch64/vplan-printing.ll | 8 ++---
10 files changed, 71 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8eb45eb3c3d00..80272f3a4e628 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1244,10 +1244,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
- ActiveLaneMaskForControlFlow,
- // Extracts each unrolled part of a widened active lane mask when used
- // for control flow.
- ExtractSubvectorForPart,
+ WideActiveLaneMask,
+ // Extracts each unrolled part of a (VF * UF) widened vector/mask.
+ ExtractVectorForPart,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 72b6503157d68..d049702c87091 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -118,7 +118,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
// the loop terminator with a branch-on-cond recipe with the negated
-// active-lane-mask as operand. Note that this turns the loop into an
+// wide-active-lane-mask as operand. Note that this turns the loop into an
// uncountable one. Only the existing terminator is replaced, all other existing
// recipes/users remain unchanged, except for poison-generating flags being
// dropped from the canonical IV increment. Return the created
@@ -128,15 +128,18 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
//
// vector.ph:
// %EntryInc = canonical-iv-increment-for-part CanonicalIVStart
-// %EntryALM = active-lane-mask %EntryInc, TC
+// %EntryALM = wide-active-lane-mask %EntryInc, TC
+// %EntryALMPart = extract-vector-for-part %EntryALM, ir<0>
//
// vector.body:
// ...
-// %P = active-lane-mask-phi [ %EntryALM, %vector.ph ], [ %ALM, %vector.body ]
+// %P = active-lane-mask-phi [ %EntryALMPart, %vector.ph ],
+// [ %ALMPart, %vector.body ]
// ...
// %InLoopInc = canonical-iv-increment-for-part CanonicalIVIncrement
-// %ALM = active-lane-mask %InLoopInc, TC
-// %Negated = Not %ALM
+// %ALM = wide-active-lane-mask %InLoopInc, TC
+// %ALMPart = extract-vector-for-part %ALM, ir<0>
+// %Negated = Not %ALMPart
// branch-on-cond %Negated
//
static VPActiveLaneMaskPHIRecipe *
@@ -148,13 +151,14 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
// TODO: Check if dropping the flags is needed.
TopRegion->clearCanonicalIVNUW(CanonicalIVIncrement);
DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
- // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
- // we have to take unrolling into account. Each part needs to start at
- // Part * VF
+ // We can't use StartV directly in the WideActiveLaneMask
+ // VPInstruction, since we have to take unrolling into account.
+ // Each part needs to start at Part * VF
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
- // Create the ActiveLaneMask instruction using the correct start values.
+ // Create the WideActiveLaneMask instruction using the correct
+ // start values.
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
@@ -162,15 +166,18 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
{StartV, VF}, {}, DL, "index.part.next");
- // Create the active lane mask instruction in the VPlan preheader.
+ // Create the wide active lane mask instruction in the VPlan preheader.
VPValue *ALMMultiplier =
Plan.getConstantInt(TopRegion->getCanonicalIVType(), 1);
- auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ auto *EntryALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
{EntryIncrement, TC, ALMMultiplier}, DL,
"active.lane.mask.entry");
+ EntryALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {EntryALM, Plan.getConstantInt(64, 0)}, DL,
+ "extract.entry.alm.part");
// Now create the ActiveLaneMaskPhi recipe in the main loop using the
- // preheader ActiveLaneMask instruction.
+ // preheader WideActiveLaneMask instruction.
auto *LaneMaskPhi =
new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc::getUnknown());
auto *HeaderVPBB = TopRegion->getEntryBasicBlock();
@@ -183,9 +190,12 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
auto *InLoopIncrement = Builder.createOverflowingOp(
VPInstruction::CanonicalIVIncrementForPart,
{CanonicalIVIncrement, &Plan.getVF()}, {}, DL);
- auto *ALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ auto *ALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
{InLoopIncrement, TC, ALMMultiplier}, DL,
"active.lane.mask.next");
+ ALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {ALM, Plan.getConstantInt(64, 0)}, DL,
+ "extract.next.alm.part");
LaneMaskPhi->addBackedgeValue(ALM);
// Replace the original terminator with BranchOnCond. We have to invert the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f4858f0d26f7a..9a6948ac6b26a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -448,9 +448,9 @@ m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
}
template <typename Op0_t, typename Op1_t>
-inline VPInstruction_match<VPInstruction::ExtractSubvectorForPart, Op0_t, Op1_t>
-m_ExtractSubvectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
- return m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(Op0, Op1);
+inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op0_t, Op1_t>
+m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op0, Op1);
}
template <typename Op0_t>
@@ -466,12 +466,10 @@ m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
}
template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ActiveLaneMaskForControlFlow, Op0_t,
- Op1_t, Op2_t>
-m_ActiveLaneMaskForControlFlow(const Op0_t &Op0, const Op1_t &Op1,
- const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>(Op0, Op1,
- Op2);
+inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
+ Op2_t>
+m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+ return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op0, Op1, Op2);
}
inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a221c1f90f423..e3bfcedc7f27a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -505,7 +505,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
assert(Op0Ty->isIntegerTy() && "expected integer operand");
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
@@ -667,12 +667,12 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::ResumeForEpilogue:
- case VPInstruction::ExtractSubvectorForPart:
+ case VPInstruction::ExtractVectorForPart:
return 2;
case Instruction::InsertElement:
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
return 3;
case Instruction::Call:
@@ -797,7 +797,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Name);
}
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow: {
+ case VPInstruction::WideActiveLaneMask: {
// Get first lane of vector induction variable.
Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
// Get the original loop tripcount.
@@ -1116,7 +1116,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
- case VPInstruction::ExtractSubvectorForPart: {
+ case VPInstruction::ExtractVectorForPart: {
Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
@@ -1129,8 +1129,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
VPValue *Start, *TC;
if (State.VF.isScalar() &&
match(getOperand(0),
- m_ActiveLaneMaskForControlFlow(m_VPValue(Start), m_VPValue(TC),
- m_VPValue()))) {
+ m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
@@ -1450,7 +1450,7 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
cast<VectorType>(VectorTy), {}, Ctx.CostKind, -1);
}
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow: {
+ case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
@@ -1662,14 +1662,14 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLastActive:
- case VPInstruction::ExtractSubvectorForPart:
+ case VPInstruction::ExtractVectorForPart:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -1720,7 +1720,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return vputils::onlyFirstLaneUsed(this);
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -1793,8 +1793,8 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ActiveLaneMask:
O << "active lane mask";
break;
- case VPInstruction::ActiveLaneMaskForControlFlow:
- O << "active lane mask control flow";
+ case VPInstruction::WideActiveLaneMask:
+ O << "wide active lane mask";
break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
@@ -1847,8 +1847,8 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
- case VPInstruction::ExtractSubvectorForPart:
- O << "extract-subvector-for-part";
+ case VPInstruction::ExtractVectorForPart:
+ O << "extract-vector-for-part";
break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 205d49c5450a3..79c257e52c7d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2054,13 +2054,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
m_VPValue())) ||
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
m_VPValue(), m_VPValue(), m_VPValue())))) ||
- match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
- m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue()),
- m_ZeroInt()))))) {
+ match(Term,
+ m_BranchOnCond(m_Not(m_ExtractVectorForPart(
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
// latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
- // BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMaskForControlFlow),
+ // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask),
// 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 45fdc1180ac50..f3d3435962d8e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -332,7 +332,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
- if (match(&R, m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(
+ if (match(&R, m_VPInstruction<VPInstruction::ExtractVectorForPart>(
m_VPValue(Op), m_VPValue()))) {
Copy->setOperand(0, Op);
Copy->setOperand(1, Plan.getConstantInt(64, Part));
@@ -477,8 +477,8 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
- if (match(&R, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue()))) {
+ if (match(&R,
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
addUniformForAllParts(ALM);
ALM->setOperand(2, Plan.getConstantInt(64, UF));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 5f124bb6e7ccd..f1316b960aeac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -197,13 +197,12 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
// (an intrinsic or a phi), or the icmp checked below.
if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()) ||
- match(Mask,
- m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>()))
+ match(Mask, m_VPInstruction<VPInstruction::WideActiveLaneMask>()))
continue;
if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
- match(Mask, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue())))
+ match(Mask,
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
continue;
CmpPredicate Pred;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 85e15c78c0ce0..b7a1e4e7924be 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -19,8 +19,8 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask control flow vp<[[VF_PER_PART]]>, vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-vector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
@@ -34,8 +34,8 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask control flow vp<[[INC]]>, vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = wide active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-vector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
; VPLANS-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 3226ed2ba779c..c40aad62202dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -25,8 +25,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
-; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow ir<0>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
@@ -41,8 +41,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<%index.next>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index de30874ee933d..ee2fac03123b9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -137,8 +137,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow vp<%index.part.next>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -157,8 +157,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<[[PART_IDX]]>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
; CHECK-NEXT: No successors
>From 5d3e353f8b5abdac929547a0b21823fb51eadda1 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 22 Jul 2026 08:55:59 +0000
Subject: [PATCH 06/14] - Removed Multiplier from existing ActiveLaneMask
VPInstruction - Moved ICMP optimisation to optimizeForVFAndUF - Remove
preferTailFoldedLoop from selectInterleaveCount() - Add test with i32 trip
count
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +-
.../Transforms/Vectorize/VPlanLowering.cpp | 19 ++-
.../Transforms/Vectorize/VPlanPatternMatch.h | 6 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 32 ++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 53 ++++++--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
.../Transforms/Vectorize/VPlanVerifier.cpp | 5 -
.../AArch64/fixed-wide-lane-mask.ll | 115 ++++++++++++++++--
.../LoopVectorize/AArch64/masked-call.ll | 2 -
.../LoopVectorize/AArch64/pr73894.ll | 5 +-
.../AArch64/scalable-strict-fadd.ll | 99 ++++++++-------
.../AArch64/uniform-args-call-variants.ll | 78 ++++++------
.../AArch64/wide-alm-scalar-vf.ll | 71 +++++++++++
.../VPlan/buildvector-first-lane-only.ll | 2 +-
.../VPlan/early_exit_with_stores_vplan.ll | 6 +-
.../Vectorize/VPlanVerifierTest.cpp | 42 +++++++
17 files changed, 378 insertions(+), 168 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e902dcbb11ceb..0647644515b94 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3640,7 +3640,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Do not interleave tail-folded loops, as the overhead of multiple
// instructions to calculate the predicate is likely not beneficial.
// If an epilogue is not allowed for any other reason, do not interleave.
- if (!CM.isEpilogueAllowed() || CM.preferTailFoldedLoop())
+ if (!CM.isEpilogueAllowed())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 80272f3a4e628..5fa10fe65eb4e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1242,8 +1242,13 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
- // The size of the mask returned is VF * Multiplier (UF, third op).
+ // The size of the mask returned is VF.
+ // When unrolled, ActiveLaneMask is duplicated.
ActiveLaneMask,
+ // As above, but takes an additional operand (Multiplier). The size of
+ // the mask returned is VF * Multiplier (UF, op2).
+ // WideActiveLaneMask is used for control flow and is unrolled by widening,
+ // with one extract vector created per unroll part.
WideActiveLaneMask,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index d049702c87091..46f16d1cc6665 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -151,17 +151,15 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
// TODO: Check if dropping the flags is needed.
TopRegion->clearCanonicalIVNUW(CanonicalIVIncrement);
DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
- // We can't use StartV directly in the WideActiveLaneMask
- // VPInstruction, since we have to take unrolling into account.
- // Each part needs to start at Part * VF
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
-
- // Create the WideActiveLaneMask instruction using the correct
- // start values.
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
+ // We can't use StartV directly in the WideActiveLaneMask
+ // VPInstruction, since we have to take unrolling into account.
+ // CanonicalIVIncrementForPart is needed so that ExtractVectorForPart
+ // is unrolled for each part, rather than each extract starting at 0.
auto *EntryIncrement =
Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
{StartV, VF}, {}, DL, "index.part.next");
@@ -225,12 +223,9 @@ void VPlanTransforms::materializeHeaderMask(
VPIRFlags::WrapFlagsTy(/*HasNUW=*/true, /*HasNSW=*/false)));
VPValue *Mask;
if (UseActiveLaneMask) {
- VPValue *ALMMultiplier =
- Plan.getConstantInt(LoopRegion->getCanonicalIVType(), 1);
- Mask = Builder.createNaryOp(
- VPInstruction::ActiveLaneMask,
- {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier}, nullptr,
- "active.lane.mask");
+ Mask = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ {WideCanonicalIV, Plan.getTripCount()}, nullptr,
+ "active.lane.mask");
} else {
Mask = Builder.createICmp(CmpInst::ICMP_ULE, WideCanonicalIV,
Plan.getOrCreateBackedgeTakenCount());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 9a6948ac6b26a..648ffa2b11a4f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -459,12 +459,6 @@ m_ExtractPenultimateElement(const Op0_t &Op0) {
return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
}
-template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t>
-m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
-}
-
template <typename Op0_t, typename Op1_t, typename Op2_t>
inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e3bfcedc7f27a..08cdc8e31274c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -657,6 +657,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::FCmp:
case Instruction::ExtractElement:
case Instruction::Store:
+ case VPInstruction::ActiveLaneMask:
case VPInstruction::BranchOnCount:
case VPInstruction::BranchOnTwoConds:
case VPInstruction::FirstOrderRecurrenceSplice:
@@ -671,7 +672,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
return 2;
case Instruction::InsertElement:
case Instruction::Select:
- case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
return 3;
@@ -803,9 +803,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
// Get the original loop tripcount.
Value *ScalarTC = State.get(getOperand(1), VPLane(0));
+ unsigned Multiplier =
+ getOpcode() == VPInstruction::ActiveLaneMask
+ ? 1
+ : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
- unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
if (State.VF.isScalar() && Multiplier == 1)
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC,
Name);
@@ -1124,23 +1128,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
if (Src->getType() == DstTy)
return Src;
- // If the VF is scalar & this is an extract of an active lane mask,
- // generate an ICMP directly.
- VPValue *Start, *TC;
- if (State.VF.isScalar() &&
- match(getOperand(0),
- m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
- m_VPValue()))) {
- Value *StartV = State.get(Start);
- if (Part > 0)
- StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
- return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
- State.get(TC));
- }
-
- auto *Idx = ConstantInt::get(Builder.getInt64Ty(),
- State.VF.getKnownMinValue() * Part);
- return Builder.CreateExtractVector(DstTy, Src, Idx);
+ return Builder.CreateExtractVector(
+ DstTy, Src, Builder.getInt64(State.VF.getKnownMinValue() * Part), Name);
}
default:
llvm_unreachable("Unsupported opcode for instruction");
@@ -1452,7 +1441,10 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
- unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ unsigned Multiplier =
+ getOpcode() == VPInstruction::ActiveLaneMask
+ ? 1
+ : cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 79c257e52c7d8..b0916fce9a8c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2037,6 +2037,43 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
+static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
+ if (!BestVF.isScalar())
+ return false;
+
+ bool MadeChange = false;
+ VPBuilder Builder;
+ VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
+ VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
+
+ VPValue *Start, *TC;
+ uint64_t Idx;
+ for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (!match(&R, m_ExtractVectorForPart(
+ m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()),
+ m_ConstantInt(Idx))))
+ continue;
+
+ auto *Extract = cast<VPInstruction>(&R);
+ Builder.setInsertPoint(Extract);
+
+ if (Idx > 0)
+ Start = Builder.createAdd(
+ Start, Plan.getConstantInt(Start->getScalarType(), Idx));
+
+ VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC);
+ Extract->replaceAllUsesWith(ICmp);
+ Extract->eraseFromParent();
+ MadeChange = true;
+ }
+ }
+
+ return MadeChange;
+}
+
/// Try to simplify the branch condition of \p Plan. This may restrict the
/// resulting plan to \p BestVF and \p BestUF.
static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
@@ -2052,16 +2089,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
if (match(Term, m_BranchOnCount(
m_CombineOr(m_CanIVInc, m_c_Add(m_CanIVInc, m_LiveIn())),
m_VPValue())) ||
- match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue())))) ||
match(Term,
m_BranchOnCond(m_Not(m_ExtractVectorForPart(
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
- // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask),
- // 0))
+ // latch terminator is BranchOnCount or
+ // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
@@ -2105,8 +2139,8 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange =
- simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
+ bool MadeChange = replaceMaskWithCompare(Plan, BestVF);
+ MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
if (MadeChange) {
@@ -3106,13 +3140,12 @@ static bool handleUncountableExitsWithSideEffects(
VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
VPValue *FirstActive = MaskBuilder.createFirstActiveLane(*Cond);
Type *IVScalarTy = IV->getScalarType();
- VPValue *ALMMultiplier = Plan.getConstantInt(IVScalarTy, 1);
VPValue *Zero = Plan.getZero(IVScalarTy);
FirstActive =
MaskBuilder.createScalarZExtOrTrunc(FirstActive, IVScalarTy, DebugLoc());
VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
- {Zero, FirstActive, ALMMultiplier},
- DebugLoc(), "uncountable.exit.mask");
+ {Zero, FirstActive}, DebugLoc(),
+ "uncountable.exit.mask");
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f3d3435962d8e..2b7a743abb6b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -481,7 +481,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
addUniformForAllParts(ALM);
- ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ ALM->setOperand(2, getConstantInt(UF));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index f1316b960aeac..73e020c636f28 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -200,11 +200,6 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
match(Mask, m_VPInstruction<VPInstruction::WideActiveLaneMask>()))
continue;
- if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
- match(Mask,
- m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
- continue;
-
CmpPredicate Pred;
VPValue *LHS, *RHS;
if (match(Mask, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) &&
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index bff2a61711124..090cb47dd315e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
@@ -84,14 +84,107 @@ for.end:
ret void
}
-attributes #0 = { nounwind "target-features"="+neon,+sve" }
+define void @fixed_wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 {
+; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask_i32_tc(
+; CHECK-UF1-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF1-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF1-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF1: vector.memcheck:
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF1-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; CHECK-UF1-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[N]])
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF1: middle.block:
+;
+; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask_i32_tc(
+; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF4: vector.memcheck:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 127
+; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 [[N]])
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 4
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 8
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 12
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP6]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK3]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP8]], <4 x i1> [[ACTIVE_LANE_MASK4]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP9]], <4 x i1> [[ACTIVE_LANE_MASK5]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD7]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD8]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 4
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 8
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 12
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr align 8 [[TMP14]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr align 8 [[TMP15]], <4 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP12]], ptr align 8 [[TMP16]], <4 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP13]], ptr align 8 [[TMP17]], <4 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-UF4-NEXT: [[TMP18]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP19]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP20]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-UF4-NEXT: [[TMP21]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
+; CHECK-UF4-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0
+; CHECK-UF4-NEXT: [[TMP23:%.*]] = xor i1 [[TMP22]], true
+; CHECK-UF4-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ br label %for.body
-;.
-; CHECK-UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-UF1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-UF1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-;.
-; CHECK-UF4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-UF4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-UF4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-;.
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind "target-features"="+neon,+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 631fed7ef89f2..2bfbe87b307d2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -996,7 +996,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1019,7 +1018,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index a01923d1ce361..ce49f94a4d43d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -12,13 +12,11 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ true, [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT7:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE5]] ]
@@ -44,7 +42,6 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]]
; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index 18a555cbc3735..a568b7152a02d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -300,21 +300,22 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP18]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP19]], %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -331,13 +332,11 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP16]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP14]], <vscale x 8 x float> [[TMP15]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT10]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP17]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP18]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP19]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP22]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP23]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -1304,21 +1303,22 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -1347,13 +1347,11 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP16]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], <vscale x 8 x float> [[TMP23]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -1551,21 +1549,22 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -1594,13 +1593,11 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP16]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], <vscale x 8 x float> [[TMP23]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index 0a5a00af05e9d..91ff96cd7f2d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -43,30 +43,30 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
-; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -127,30 +127,30 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
-; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -198,7 +198,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: entry:
; INTERLEAVE-NEXT: br label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N]])
; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]]
; INTERLEAVE-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]]
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -226,7 +225,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE3]]
; INTERLEAVE: pred.store.continue3:
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; INTERLEAVE-NEXT: [[TMP11]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i64 [[INDEX_NEXT]], 1
; INTERLEAVE-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
new file mode 100644
index 0000000000000..f6174a0ee37bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=1 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4
+
+target triple = "aarch64-unknown-linux"
+
+define void @wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 {
+; CHECK-UF4-LABEL: define void @wide_active_lane_mask_i32_tc(
+; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF4: vector.memcheck:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15
+; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = icmp ult i32 0, [[N]]
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = icmp ult i32 1, [[N]]
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE5:%.*]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE5]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi i1 [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_STORE_CONTINUE5]] ]
+; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK-UF4: pred.store.if:
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP4]], align 8
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = fmul double [[TMP5]], 3.000000e+00
+; CHECK-UF4-NEXT: store double [[TMP7]], ptr [[TMP6]], align 8
+; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE]]
+; CHECK-UF4: pred.store.continue:
+; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK3]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5]]
+; CHECK-UF4: pred.store.if4:
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP9]], align 8
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul double [[TMP10]], 3.000000e+00
+; CHECK-UF4-NEXT: store double [[TMP12]], ptr [[TMP11]], align 8
+; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE5]]
+; CHECK-UF4: pred.store.continue5:
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-UF4-NEXT: [[TMP13]] = icmp ult i32 [[INDEX_NEXT]], [[N]]
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = add i32 [[INDEX_NEXT]], 1
+; CHECK-UF4-NEXT: [[TMP15]] = icmp ult i32 [[TMP14]], [[N]]
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = xor i1 [[TMP13]], true
+; CHECK-UF4-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index ded3bc66016fe..b57af70695f6c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -17,7 +17,7 @@ define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index d3b9982f45419..70bed19a17a62 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -28,7 +28,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
@@ -117,7 +117,7 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
@@ -202,7 +202,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 7ce87be841c2d..0dd1340556bea 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -612,4 +612,46 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
#endif
}
+TEST_F(VPIRVerifierTest, testVerifyLastActiveWideActiveLaneMask) {
+ const char *ModuleString =
+ "define i32 @f(ptr %A, i64 %N) {\n"
+ "entry:\n"
+ " br label %loop\n"
+ "loop:\n"
+ " %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n"
+ " %gep = getelementptr i32, ptr %A, i64 %iv\n"
+ " %ld = load i32, ptr %gep \n"
+ " %iv.next = add i64 %iv, 1\n"
+ " %exitcond = icmp eq i64 %iv.next, %N\n"
+ " br i1 %exitcond, label %exit, label %loop\n"
+ "exit:\n"
+ " ret i32 %ld\n"
+ "}\n";
+
+ Module &M = parseModule(ModuleString);
+
+ Function *F = M.getFunction("f");
+ BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
+ auto Plan = buildVPlan(LoopHeader);
+
+ VPValue *Start = Plan->getConstantInt(64, 0);
+ VPValue *TC = Plan->getTripCount();
+ VPValue *One = Plan->getConstantInt(64, 1);
+ Type *MaskTy = FixedVectorType::get(Type::getInt1Ty(*Ctx), 4);
+ Type *IndexTy = Plan->getDataLayout().getIndexType(Plan->getContext(), 0);
+
+ auto *WideALM =
+ new VPInstruction(VPInstruction::WideActiveLaneMask, {Start, TC, One}, {},
+ {}, DebugLoc(), "", MaskTy);
+ auto *Preheader = Plan->getVectorPreheader();
+ WideALM->insertBefore(*Preheader, Preheader->end());
+
+ auto *LastActiveLane =
+ new VPInstruction(VPInstruction::LastActiveLane, {WideALM}, {}, {},
+ DebugLoc(), "", IndexTy);
+ LastActiveLane->insertBefore(Plan->getMiddleBlock()->getTerminator());
+
+ EXPECT_TRUE(verifyVPlanIsValid(*Plan));
+}
+
} // namespace
>From 380e4a6e2a4ad1e15af838644dddbe0a39c911c0 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 27 Jul 2026 10:46:10 +0000
Subject: [PATCH 07/14] - Add a test with const trip count
---
.../LoopVectorize/AArch64/wide-alm-const.ll | 61 +++++++++++++++++++
1 file changed, 61 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
new file mode 100644
index 0000000000000..7243a4b56358d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=4 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4
+
+target triple = "aarch64-unknown-linux"
+
+define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonly %src) #0 {
+; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double(
+; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-UF4-NEXT: [[ENTRY:.*:]]
+; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 9, 0
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-UF4: [[FOR_BODY_PREHEADER]]:
+; CHECK-UF4-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-UF4: [[VECTOR_PH]]:
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 9)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF4: [[VECTOR_BODY]]:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 4
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK2]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD3]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP4]], i64 4
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP2]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 9)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4: [[MIDDLE_BLOCK]]:
+;
+entry:
+ %cmp6 = icmp sgt i64 9, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i64 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 9
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
>From 38af60a6c7ebd08dc02ab6fc5f8118e16fe04929 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 5 Aug 2026 12:38:45 +0000
Subject: [PATCH 08/14] - Swap checks for ActiveLaneMask with
WideActiveLaneMask in VPRecipes - Replace unsigned with uint64_t for
getZExtValue() - Add comment to describe replaceMaskWithCompare
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +++
3 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5fa10fe65eb4e..746e918a3f1df 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1242,7 +1242,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
- // The size of the mask returned is VF.
+ // ActiveLaneMask is used for tail-folding, with the exception of the
+ // DataAndControlFlow style. The size of the mask returned is VF.
// When unrolled, ActiveLaneMask is duplicated.
ActiveLaneMask,
// As above, but takes an additional operand (Multiplier). The size of
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 08cdc8e31274c..f2256442b0d0a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -803,10 +803,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
// Get the original loop tripcount.
Value *ScalarTC = State.get(getOperand(1), VPLane(0));
- unsigned Multiplier =
- getOpcode() == VPInstruction::ActiveLaneMask
- ? 1
- : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ uint64_t Multiplier =
+ getOpcode() == VPInstruction::WideActiveLaneMask
+ ? cast<VPConstantInt>(getOperand(2))->getZExtValue()
+ : 1;
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
@@ -1123,7 +1123,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::ExtractVectorForPart: {
Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
- unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
+ uint64_t Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
if (Src->getType() == DstTy)
return Src;
@@ -1441,10 +1441,10 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
- unsigned Multiplier =
- getOpcode() == VPInstruction::ActiveLaneMask
- ? 1
- : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ uint64_t Multiplier =
+ getOpcode() == VPInstruction::WideActiveLaneMask
+ ? cast<VPConstantInt>(getOperand(2))->getZExtValue()
+ : 1;
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b0916fce9a8c0..79eb6f4561f8f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2037,6 +2037,9 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
+// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
+// and the source is a WideActiveLaneMask. The unused mask is removed later
+// when removing dead recipes.
static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
if (!BestVF.isScalar())
return false;
>From 6a3a19dab45f2a6314d1c24540aac6ef50c9e6e4 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 7 Aug 2026 15:18:57 +0000
Subject: [PATCH 09/14] - Address new feedback: - Remove
CanonicalIVIncrementForPart in addVPLaneMaskPhiAndUpdateExitBranch and
ensure isUniformAcrossVFsAndUFs returns false for ExtractVectorForPart -
Use overflowing add in replaceMaskWithCompare
---
.../Transforms/Vectorize/VPlanLowering.cpp | 10 +-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +++--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 1 -
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 14 +++++------
.../AArch64/sve-wide-lane-mask.ll | 24 +++++++++----------
.../VPlan/AArch64/sve-tail-folding-forced.ll | 3 +--
.../VPlan/AArch64/vplan-printing.ll | 3 +--
7 files changed, 26 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 46f16d1cc6665..9a0c0cc17e96b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -156,19 +156,11 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
- // We can't use StartV directly in the WideActiveLaneMask
- // VPInstruction, since we have to take unrolling into account.
- // CanonicalIVIncrementForPart is needed so that ExtractVectorForPart
- // is unrolled for each part, rather than each extract starting at 0.
- auto *EntryIncrement =
- Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
- {StartV, VF}, {}, DL, "index.part.next");
-
// Create the wide active lane mask instruction in the VPlan preheader.
VPValue *ALMMultiplier =
Plan.getConstantInt(TopRegion->getCanonicalIVType(), 1);
auto *EntryALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
- {EntryIncrement, TC, ALMMultiplier}, DL,
+ {StartV, TC, ALMMultiplier}, DL,
"active.lane.mask.entry");
EntryALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
{EntryALM, Plan.getConstantInt(64, 0)}, DL,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 79eb6f4561f8f..c21c11df24296 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2064,8 +2064,10 @@ static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
Builder.setInsertPoint(Extract);
if (Idx > 0)
- Start = Builder.createAdd(
- Start, Plan.getConstantInt(Start->getScalarType(), Idx));
+ Start = Builder.createOverflowingOp(
+ Instruction::Add,
+ {Start, Plan.getConstantInt(Start->getScalarType(), Idx)},
+ {false, false});
VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC);
Extract->replaceAllUsesWith(ICmp);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 2b7a743abb6b6..21ea2a6b74ff5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -480,7 +480,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
if (match(&R,
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
- addUniformForAllParts(ALM);
ALM->setOperand(2, getConstantInt(UF));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e450b08edfb97..fe44ae5c1f846 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -465,13 +465,13 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
const VPRecipeBase *R = V->getDefiningRecipe();
const VPBasicBlock *VPBB = R ? R->getParent() : nullptr;
const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr;
- if (VPBB) {
- if ((VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
- if (match(V->getDefiningRecipe(),
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()))
- return false;
- return all_of(R->operands(), isUniformAcrossVFsAndUFs);
- }
+ if (VPBB &&
+ (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
+ if (match(R,
+ m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()) ||
+ match(R, m_ExtractVectorForPart(m_VPValue(), m_VPValue())))
+ return false;
+ return all_of(R->operands(), isUniformAcrossVFsAndUFs);
}
return TypeSwitch<const VPRecipeBase *, bool>(R)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index f77faf7eac54e..8c213d3aa528f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -40,16 +40,16 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 0, i64 [[N]])
; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
-; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
@@ -142,16 +142,16 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl
; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
-; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index b7a1e4e7924be..be39caa191d09 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -18,8 +18,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: Successor(s): scalar.ph, vector.ph
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
-; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask ir<0>, vp<[[TC]]>, ir<1>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-vector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index ee2fac03123b9..0aa28b9d37d23 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -136,8 +136,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
-; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
>From d535d6f5623c9b52bcc0d94b115a2583ff8905e8 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 7 Aug 2026 15:57:55 +0000
Subject: [PATCH 10/14] - Remove unused variable 'VF' - Remove
`InLoopIncrement` from addVPLaneMaskPhiAndUpdateExitBranch - Revert
unnecessary change to replaceMaskWithCompare
---
llvm/lib/Transforms/Vectorize/VPlanLowering.cpp | 8 ++------
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 ++----
.../VPlan/AArch64/sve-tail-folding-forced.ll | 3 +--
.../LoopVectorize/VPlan/AArch64/vplan-printing.ll | 3 +--
4 files changed, 6 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 9a0c0cc17e96b..c1687e49824ed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -154,7 +154,6 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
VPValue *TC = Plan.getTripCount();
- VPValue *VF = &Plan.getVF();
// Create the wide active lane mask instruction in the VPlan preheader.
VPValue *ALMMultiplier =
@@ -177,12 +176,9 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
// original terminator.
VPRecipeBase *OriginalTerminator = EB->getTerminator();
Builder.setInsertPoint(OriginalTerminator);
- auto *InLoopIncrement = Builder.createOverflowingOp(
- VPInstruction::CanonicalIVIncrementForPart,
- {CanonicalIVIncrement, &Plan.getVF()}, {}, DL);
auto *ALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
- {InLoopIncrement, TC, ALMMultiplier}, DL,
- "active.lane.mask.next");
+ {CanonicalIVIncrement, TC, ALMMultiplier},
+ DL, "active.lane.mask.next");
ALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
{ALM, Plan.getConstantInt(64, 0)}, DL,
"extract.next.alm.part");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c21c11df24296..79eb6f4561f8f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2064,10 +2064,8 @@ static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
Builder.setInsertPoint(Extract);
if (Idx > 0)
- Start = Builder.createOverflowingOp(
- Instruction::Add,
- {Start, Plan.getConstantInt(Start->getScalarType(), Idx)},
- {false, false});
+ Start = Builder.createAdd(
+ Start, Plan.getConstantInt(Start->getScalarType(), Idx));
VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC);
Extract->replaceAllUsesWith(ICmp);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index be39caa191d09..ddf757691047d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -32,8 +32,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer i32, ir<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
-; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = wide active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = wide active lane mask vp<[[INDV_UPDATE]]>, vp<[[TC]]>, ir<1>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-vector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 0aa28b9d37d23..6d65986047267 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -155,8 +155,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>, vp<[[MASK]]>
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
>From c51aea13fb8efe346a82769a5372fbe4105b602a Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 10 Aug 2026 13:48:38 +0000
Subject: [PATCH 11/14] - Add test for simplifyBranchConditionForVFAndUF with
known trip count
---
.../AArch64/fixed-wide-lane-mask.ll | 64 +++++++++++++++++++
1 file changed, 64 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index 090cb47dd315e..b59fc73ebbf0f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -187,4 +187,68 @@ for.end:
ret void
}
+define void @fixed_known_tc(ptr noalias readonly %dst, ptr noalias readonly %src) #0 {
+; CHECK-UF1-LABEL: define void @fixed_known_tc(
+; CHECK-UF1-SAME: ptr noalias readonly [[DST:%.*]], ptr noalias readonly [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 15)
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY1:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[IV:%.*]] = phi i64 [ 0, [[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-UF1-NEXT: [[X:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-UF1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[ARRAYIDX]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 4
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 15)
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-UF1-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-UF1: middle.block:
+;
+; CHECK-UF4-LABEL: define void @fixed_known_tc(
+; CHECK-UF4-SAME: ptr noalias readonly [[DST:%.*]], ptr noalias readonly [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 15)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[X:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-UF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-UF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 4
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 8
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 12
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[DST]], <4 x i1> [[EXTRACT_ENTRY_ALM_PART]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP1]], <4 x i1> [[EXTRACT_ENTRY_ALM_PART1]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <4 x i1> [[EXTRACT_ENTRY_ALM_PART2]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <4 x i1> [[EXTRACT_ENTRY_ALM_PART3]])
+; CHECK-UF4-NEXT: br label [[MIDDLE_BLOCK:%.*]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %x = load i32, ptr %src
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %x, ptr %gep
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 15
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
attributes #0 = { nounwind "target-features"="+neon,+sve" }
>From d4a509540e7d62859ebb47b89feef669689afef8 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 11 Aug 2026 10:54:53 +0000
Subject: [PATCH 12/14] - Remove match(Mask, WideActiveLaneMask) from
verifyLastActiveLaneRecipe - Remove changes to VPlanVerifierTest.cpp - Add
test for a loop with an outside use
---
.../Transforms/Vectorize/VPlanVerifier.cpp | 3 +-
.../AArch64/sve-wide-lane-mask.ll | 115 ++++++++++++++++++
.../Vectorize/VPlanVerifierTest.cpp | 42 -------
3 files changed, 116 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 73e020c636f28..e36ad81cfae2a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -196,8 +196,7 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
// loop region's abstract header mask; afterwards it is an active lane mask
// (an intrinsic or a phi), or the icmp checked below.
if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
- match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()) ||
- match(Mask, m_VPInstruction<VPInstruction::WideActiveLaneMask>()))
+ match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()))
continue;
CmpPredicate Pred;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 8c213d3aa528f..70b9bb9c8382b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -204,4 +204,119 @@ for.end:
ret void
}
+define i64 @scalable_wide_lane_mask_outside_use(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-UF1-LABEL: define i64 @scalable_wide_lane_mask_outside_use(
+; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF1: for.body.preheader:
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-UF1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF1-NEXT: [[TMP6]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-UF1-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UF1: middle.block:
+;
+; CHECK-UF4-LABEL: define i64 @scalable_wide_lane_mask_outside_use(
+; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF4: for.body.preheader:
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-UF4-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP15]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART10]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-UF4-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-UF4-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ %cmp6 = icmp sgt i64 %n, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %last = phi i64 [ 0, %entry ], [ %mul, %for.body ]
+ %arrayidx1 = getelementptr inbounds i64, ptr %src, i64 %iv
+ %ld = load i64, ptr %arrayidx1
+ %mul = mul i64 %ld, 3
+ %arrayidx2 = getelementptr inbounds i64, ptr %dst, i64 %iv
+ store i64 %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ %res = phi i64 [0, %entry], [ %last, %for.body]
+ ret i64 %res
+}
+
attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 0dd1340556bea..7ce87be841c2d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -612,46 +612,4 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
#endif
}
-TEST_F(VPIRVerifierTest, testVerifyLastActiveWideActiveLaneMask) {
- const char *ModuleString =
- "define i32 @f(ptr %A, i64 %N) {\n"
- "entry:\n"
- " br label %loop\n"
- "loop:\n"
- " %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n"
- " %gep = getelementptr i32, ptr %A, i64 %iv\n"
- " %ld = load i32, ptr %gep \n"
- " %iv.next = add i64 %iv, 1\n"
- " %exitcond = icmp eq i64 %iv.next, %N\n"
- " br i1 %exitcond, label %exit, label %loop\n"
- "exit:\n"
- " ret i32 %ld\n"
- "}\n";
-
- Module &M = parseModule(ModuleString);
-
- Function *F = M.getFunction("f");
- BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
- auto Plan = buildVPlan(LoopHeader);
-
- VPValue *Start = Plan->getConstantInt(64, 0);
- VPValue *TC = Plan->getTripCount();
- VPValue *One = Plan->getConstantInt(64, 1);
- Type *MaskTy = FixedVectorType::get(Type::getInt1Ty(*Ctx), 4);
- Type *IndexTy = Plan->getDataLayout().getIndexType(Plan->getContext(), 0);
-
- auto *WideALM =
- new VPInstruction(VPInstruction::WideActiveLaneMask, {Start, TC, One}, {},
- {}, DebugLoc(), "", MaskTy);
- auto *Preheader = Plan->getVectorPreheader();
- WideALM->insertBefore(*Preheader, Preheader->end());
-
- auto *LastActiveLane =
- new VPInstruction(VPInstruction::LastActiveLane, {WideALM}, {}, {},
- DebugLoc(), "", IndexTy);
- LastActiveLane->insertBefore(Plan->getMiddleBlock()->getTerminator());
-
- EXPECT_TRUE(verifyVPlanIsValid(*Plan));
-}
-
} // namespace
>From 764042d83b11206e599ace33376c5386ffd47347 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 11 Aug 2026 12:35:42 +0000
Subject: [PATCH 13/14] - Move new test into new sve-wide-mask-outside-use.ll
file - Add new test with an outside use of a load instruction
---
.../AArch64/sve-wide-lane-mask.ll | 115 ------
.../AArch64/sve-wide-mask-outside-use.ll | 364 ++++++++++++++++++
2 files changed, 364 insertions(+), 115 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 70b9bb9c8382b..8c213d3aa528f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -204,119 +204,4 @@ for.end:
ret void
}
-define i64 @scalable_wide_lane_mask_outside_use(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
-; CHECK-UF1-LABEL: define i64 @scalable_wide_lane_mask_outside_use(
-; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-UF1: for.body.preheader:
-; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-UF1-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
-; CHECK-UF1-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
-; CHECK-UF1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP4]]
-; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-UF1: vector.body:
-; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; CHECK-UF1-NEXT: [[TMP6]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
-; CHECK-UF1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-UF1-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
-; CHECK-UF1-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-UF1: middle.block:
-;
-; CHECK-UF4-LABEL: define i64 @scalable_wide_lane_mask_outside_use(
-; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-UF4: for.body.preheader:
-; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-UF4: vector.ph:
-; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
-; CHECK-UF4-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
-; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-UF4: vector.body:
-; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART11:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP1]], 1
-; CHECK-UF4-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[TMP1]], 3
-; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]]
-; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP15]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP7]]
-; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP8]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
-; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART10]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
-; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
-; CHECK-UF4-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
-; CHECK-UF4-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
-; CHECK-UF4-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-UF4: middle.block:
-;
-entry:
- %cmp6 = icmp sgt i64 %n, 0
- br i1 %cmp6, label %for.body, label %for.end
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %last = phi i64 [ 0, %entry ], [ %mul, %for.body ]
- %arrayidx1 = getelementptr inbounds i64, ptr %src, i64 %iv
- %ld = load i64, ptr %arrayidx1
- %mul = mul i64 %ld, 3
- %arrayidx2 = getelementptr inbounds i64, ptr %dst, i64 %iv
- store i64 %mul, ptr %arrayidx2
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond.not = icmp eq i64 %iv.next, %n
- br i1 %exitcond.not, label %for.end, label %for.body
-
-for.end:
- %res = phi i64 [0, %entry], [ %last, %for.body]
- ret i64 %res
-}
-
attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
new file mode 100644
index 0000000000000..38841f8d10495
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
@@ -0,0 +1,364 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
+; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
+
+target triple = "aarch64-unknown-linux"
+
+define i64 @scalable_wide_lane_mask_outside_use(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-UF1-LABEL: @scalable_wide_lane_mask_outside_use(
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF1: for.body.preheader:
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-UF1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF1-NEXT: [[TMP6]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-UF1-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF1: middle.block:
+; CHECK-UF1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP6]], i32 1)
+; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-UF1-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; CHECK-UF1-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-UF1-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i64> [[TMP10]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF1-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-UF1: for.end.loopexit:
+; CHECK-UF1-NEXT: br label [[FOR_END]]
+; CHECK-UF1: for.end:
+; CHECK-UF1-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP12]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-UF1-NEXT: ret i64 [[RES]]
+;
+; CHECK-UF4-LABEL: @scalable_wide_lane_mask_outside_use(
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF4: for.body.preheader:
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-UF4-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP15]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART10]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-UF4-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-UF4-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4: middle.block:
+; CHECK-UF4-NEXT: [[TMP22:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP12]], i32 1)
+; CHECK-UF4-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]], i32 1)
+; CHECK-UF4-NEXT: [[TMP24:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[TMP13]], <vscale x 2 x i64> [[TMP14]], i32 1)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], i32 1)
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP27:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP28:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP29:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 2
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP29]], i1 false)
+; CHECK-UF4-NEXT: [[TMP32:%.*]] = mul i64 [[TMP31]], 3
+; CHECK-UF4-NEXT: [[TMP33:%.*]] = add i64 [[TMP32]], [[FIRST_INACTIVE_LANE]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP28]], i1 false)
+; CHECK-UF4-NEXT: [[TMP34:%.*]] = mul i64 [[TMP31]], 2
+; CHECK-UF4-NEXT: [[TMP35:%.*]] = add i64 [[TMP34]], [[FIRST_INACTIVE_LANE13]]
+; CHECK-UF4-NEXT: [[TMP36:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE13]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP37:%.*]] = select i1 [[TMP36]], i64 [[TMP35]], i64 [[TMP33]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP27]], i1 false)
+; CHECK-UF4-NEXT: [[TMP38:%.*]] = mul i64 [[TMP31]], 1
+; CHECK-UF4-NEXT: [[TMP39:%.*]] = add i64 [[TMP38]], [[FIRST_INACTIVE_LANE14]]
+; CHECK-UF4-NEXT: [[TMP40:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE14]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP37]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP26]], i1 false)
+; CHECK-UF4-NEXT: [[TMP42:%.*]] = mul i64 [[TMP31]], 0
+; CHECK-UF4-NEXT: [[TMP43:%.*]] = add i64 [[TMP42]], [[FIRST_INACTIVE_LANE15]]
+; CHECK-UF4-NEXT: [[TMP44:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE15]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i64 [[TMP43]], i64 [[TMP41]]
+; CHECK-UF4-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP45]], 1
+; CHECK-UF4-NEXT: [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP47:%.*]] = mul nuw i64 [[TMP46]], 2
+; CHECK-UF4-NEXT: [[TMP48:%.*]] = mul i64 [[TMP47]], 0
+; CHECK-UF4-NEXT: [[TMP49:%.*]] = extractelement <vscale x 2 x i64> [[TMP22]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF4-NEXT: [[TMP50:%.*]] = mul i64 [[TMP47]], 1
+; CHECK-UF4-NEXT: [[TMP51:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
+; CHECK-UF4-NEXT: [[TMP52:%.*]] = extractelement <vscale x 2 x i64> [[TMP23]], i64 [[TMP51]]
+; CHECK-UF4-NEXT: [[TMP53:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
+; CHECK-UF4-NEXT: [[TMP54:%.*]] = select i1 [[TMP53]], i64 [[TMP52]], i64 [[TMP49]]
+; CHECK-UF4-NEXT: [[TMP55:%.*]] = mul i64 [[TMP47]], 2
+; CHECK-UF4-NEXT: [[TMP56:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
+; CHECK-UF4-NEXT: [[TMP57:%.*]] = extractelement <vscale x 2 x i64> [[TMP24]], i64 [[TMP56]]
+; CHECK-UF4-NEXT: [[TMP58:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
+; CHECK-UF4-NEXT: [[TMP59:%.*]] = select i1 [[TMP58]], i64 [[TMP57]], i64 [[TMP54]]
+; CHECK-UF4-NEXT: [[TMP60:%.*]] = mul i64 [[TMP47]], 3
+; CHECK-UF4-NEXT: [[TMP61:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
+; CHECK-UF4-NEXT: [[TMP62:%.*]] = extractelement <vscale x 2 x i64> [[TMP25]], i64 [[TMP61]]
+; CHECK-UF4-NEXT: [[TMP63:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
+; CHECK-UF4-NEXT: [[TMP64:%.*]] = select i1 [[TMP63]], i64 [[TMP62]], i64 [[TMP59]]
+; CHECK-UF4-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-UF4: for.end.loopexit:
+; CHECK-UF4-NEXT: br label [[FOR_END]]
+; CHECK-UF4: for.end:
+; CHECK-UF4-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP64]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-UF4-NEXT: ret i64 [[RES]]
+;
+entry:
+ %cmp6 = icmp sgt i64 %n, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %last = phi i64 [ 0, %entry ], [ %mul, %for.body ]
+ %arrayidx1 = getelementptr inbounds i64, ptr %src, i64 %iv
+ %ld = load i64, ptr %arrayidx1
+ %mul = mul i64 %ld, 3
+ %arrayidx2 = getelementptr inbounds i64, ptr %dst, i64 %iv
+ store i64 %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ %res = phi i64 [0, %entry], [ %last, %for.body]
+ ret i64 %res
+}
+
+define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-UF1-LABEL: @scalable_wide_lane_mask_outside_load_use(
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF1: for.body.preheader:
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-UF1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF1-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-UF1-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF1: middle.block:
+; CHECK-UF1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-UF1-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; CHECK-UF1-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-UF1-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i64> [[TMP10]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF1-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-UF1: for.end.loopexit:
+; CHECK-UF1-NEXT: br label [[FOR_END]]
+; CHECK-UF1: for.end:
+; CHECK-UF1-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP12]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-UF1-NEXT: ret i64 [[RES]]
+;
+; CHECK-UF4-LABEL: @scalable_wide_lane_mask_outside_load_use(
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK-UF4: for.body.preheader:
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-UF4-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 3)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP7]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART10]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-UF4-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-UF4-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF4: middle.block:
+; CHECK-UF4-NEXT: [[TMP22:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-UF4-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], i32 1)
+; CHECK-UF4-NEXT: [[TMP24:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], i32 1)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], i32 1)
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP27:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP28:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP29:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], splat (i1 true)
+; CHECK-UF4-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 2
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP29]], i1 false)
+; CHECK-UF4-NEXT: [[TMP32:%.*]] = mul i64 [[TMP31]], 3
+; CHECK-UF4-NEXT: [[TMP33:%.*]] = add i64 [[TMP32]], [[FIRST_INACTIVE_LANE]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP28]], i1 false)
+; CHECK-UF4-NEXT: [[TMP34:%.*]] = mul i64 [[TMP31]], 2
+; CHECK-UF4-NEXT: [[TMP35:%.*]] = add i64 [[TMP34]], [[FIRST_INACTIVE_LANE13]]
+; CHECK-UF4-NEXT: [[TMP36:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE13]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP37:%.*]] = select i1 [[TMP36]], i64 [[TMP35]], i64 [[TMP33]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP27]], i1 false)
+; CHECK-UF4-NEXT: [[TMP38:%.*]] = mul i64 [[TMP31]], 1
+; CHECK-UF4-NEXT: [[TMP39:%.*]] = add i64 [[TMP38]], [[FIRST_INACTIVE_LANE14]]
+; CHECK-UF4-NEXT: [[TMP40:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE14]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP37]]
+; CHECK-UF4-NEXT: [[FIRST_INACTIVE_LANE15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP26]], i1 false)
+; CHECK-UF4-NEXT: [[TMP42:%.*]] = mul i64 [[TMP31]], 0
+; CHECK-UF4-NEXT: [[TMP43:%.*]] = add i64 [[TMP42]], [[FIRST_INACTIVE_LANE15]]
+; CHECK-UF4-NEXT: [[TMP44:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE15]], [[TMP31]]
+; CHECK-UF4-NEXT: [[TMP45:%.*]] = select i1 [[TMP44]], i64 [[TMP43]], i64 [[TMP41]]
+; CHECK-UF4-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP45]], 1
+; CHECK-UF4-NEXT: [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP47:%.*]] = mul nuw i64 [[TMP46]], 2
+; CHECK-UF4-NEXT: [[TMP48:%.*]] = mul i64 [[TMP47]], 0
+; CHECK-UF4-NEXT: [[TMP49:%.*]] = extractelement <vscale x 2 x i64> [[TMP22]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF4-NEXT: [[TMP50:%.*]] = mul i64 [[TMP47]], 1
+; CHECK-UF4-NEXT: [[TMP51:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
+; CHECK-UF4-NEXT: [[TMP52:%.*]] = extractelement <vscale x 2 x i64> [[TMP23]], i64 [[TMP51]]
+; CHECK-UF4-NEXT: [[TMP53:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
+; CHECK-UF4-NEXT: [[TMP54:%.*]] = select i1 [[TMP53]], i64 [[TMP52]], i64 [[TMP49]]
+; CHECK-UF4-NEXT: [[TMP55:%.*]] = mul i64 [[TMP47]], 2
+; CHECK-UF4-NEXT: [[TMP56:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
+; CHECK-UF4-NEXT: [[TMP57:%.*]] = extractelement <vscale x 2 x i64> [[TMP24]], i64 [[TMP56]]
+; CHECK-UF4-NEXT: [[TMP58:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
+; CHECK-UF4-NEXT: [[TMP59:%.*]] = select i1 [[TMP58]], i64 [[TMP57]], i64 [[TMP54]]
+; CHECK-UF4-NEXT: [[TMP60:%.*]] = mul i64 [[TMP47]], 3
+; CHECK-UF4-NEXT: [[TMP61:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
+; CHECK-UF4-NEXT: [[TMP62:%.*]] = extractelement <vscale x 2 x i64> [[TMP25]], i64 [[TMP61]]
+; CHECK-UF4-NEXT: [[TMP63:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
+; CHECK-UF4-NEXT: [[TMP64:%.*]] = select i1 [[TMP63]], i64 [[TMP62]], i64 [[TMP59]]
+; CHECK-UF4-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK-UF4: for.end.loopexit:
+; CHECK-UF4-NEXT: br label [[FOR_END]]
+; CHECK-UF4: for.end:
+; CHECK-UF4-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP64]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-UF4-NEXT: ret i64 [[RES]]
+;
+entry:
+ %cmp6 = icmp sgt i64 %n, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %last = phi i64 [ 0, %entry ], [ %load, %for.body ]
+ %arrayidx1 = getelementptr inbounds i64, ptr %src, i64 %iv
+ %load = load i64, ptr %arrayidx1
+ %mul = mul i64 %load, 3
+ %arrayidx2 = getelementptr inbounds i64, ptr %dst, i64 %iv
+ store i64 %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ %res = phi i64 [0, %entry], [ %last, %for.body]
+ ret i64 %res
+}
+
+attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
>From 8ea1f3ba822daa3a26b52349fbb93ba5ec151a84 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 11 Aug 2026 13:32:20 +0000
Subject: [PATCH 14/14] - Simplify scalable_wide_lane_mask_outside_load_use
test
---
.../AArch64/sve-wide-mask-outside-use.ll | 76 +++++--------------
1 file changed, 20 insertions(+), 56 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
index 38841f8d10495..7055f619c1301 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-mask-outside-use.ll
@@ -186,65 +186,45 @@ for.end:
define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
; CHECK-UF1-LABEL: @scalable_wide_lane_mask_outside_load_use(
; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
-; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-UF1: for.body.preheader:
; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-UF1-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
-; CHECK-UF1-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
-; CHECK-UF1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N:%.*]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; CHECK-UF1-NEXT: [[TMP6:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; CHECK-UF1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-UF1-NEXT: [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF1-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
; CHECK-UF1-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF1: middle.block:
-; CHECK-UF1-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i32 1)
; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
; CHECK-UF1-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
; CHECK-UF1-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-UF1-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i64> [[TMP10]], i64 [[LAST_ACTIVE_LANE]]
-; CHECK-UF1-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
-; CHECK-UF1: for.end.loopexit:
-; CHECK-UF1-NEXT: br label [[FOR_END]]
+; CHECK-UF1-NEXT: [[RES:%.*]] = extractelement <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF1-NEXT: br label [[FOR_END:%.*]]
; CHECK-UF1: for.end:
-; CHECK-UF1-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP12]], [[FOR_END_LOOPEXIT]] ]
; CHECK-UF1-NEXT: ret i64 [[RES]]
;
; CHECK-UF4-LABEL: @scalable_wide_lane_mask_outside_load_use(
; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N:%.*]], 0
-; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-UF4: for.body.preheader:
; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N:%.*]])
; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
-; CHECK-UF4-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 0, i32 [[TMP5]]
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -252,7 +232,6 @@ define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr reado
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART10:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART11:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD9:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP1]], 1
; CHECK-UF4-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[TMP1]], 3
@@ -262,19 +241,15 @@ define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr reado
; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], <vscale x 2 x i64> poison)
; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
-; CHECK-UF4-NEXT: [[TMP12:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 3)
-; CHECK-UF4-NEXT: [[TMP15:%.*]] = mul <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 3)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> poison)
; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP1]]
; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP7]]
; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i64 [[TMP8]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP16]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
@@ -285,10 +260,6 @@ define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr reado
; CHECK-UF4-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
; CHECK-UF4-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF4: middle.block:
-; CHECK-UF4-NEXT: [[TMP22:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i32 1)
-; CHECK-UF4-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], i32 1)
-; CHECK-UF4-NEXT: [[TMP24:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], i32 1)
-; CHECK-UF4-NEXT: [[TMP25:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], i32 1)
; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
; CHECK-UF4-NEXT: [[TMP27:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK4]], splat (i1 true)
; CHECK-UF4-NEXT: [[TMP28:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], splat (i1 true)
@@ -317,48 +288,41 @@ define i64 @scalable_wide_lane_mask_outside_load_use(ptr noalias %dst, ptr reado
; CHECK-UF4-NEXT: [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF4-NEXT: [[TMP47:%.*]] = mul nuw i64 [[TMP46]], 2
; CHECK-UF4-NEXT: [[TMP48:%.*]] = mul i64 [[TMP47]], 0
-; CHECK-UF4-NEXT: [[TMP49:%.*]] = extractelement <vscale x 2 x i64> [[TMP22]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-UF4-NEXT: [[TMP49:%.*]] = extractelement <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-UF4-NEXT: [[TMP50:%.*]] = mul i64 [[TMP47]], 1
; CHECK-UF4-NEXT: [[TMP51:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
-; CHECK-UF4-NEXT: [[TMP52:%.*]] = extractelement <vscale x 2 x i64> [[TMP23]], i64 [[TMP51]]
+; CHECK-UF4-NEXT: [[TMP52:%.*]] = extractelement <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], i64 [[TMP51]]
; CHECK-UF4-NEXT: [[TMP53:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP50]]
; CHECK-UF4-NEXT: [[TMP54:%.*]] = select i1 [[TMP53]], i64 [[TMP52]], i64 [[TMP49]]
; CHECK-UF4-NEXT: [[TMP55:%.*]] = mul i64 [[TMP47]], 2
; CHECK-UF4-NEXT: [[TMP56:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
-; CHECK-UF4-NEXT: [[TMP57:%.*]] = extractelement <vscale x 2 x i64> [[TMP24]], i64 [[TMP56]]
+; CHECK-UF4-NEXT: [[TMP57:%.*]] = extractelement <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], i64 [[TMP56]]
; CHECK-UF4-NEXT: [[TMP58:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP55]]
; CHECK-UF4-NEXT: [[TMP59:%.*]] = select i1 [[TMP58]], i64 [[TMP57]], i64 [[TMP54]]
; CHECK-UF4-NEXT: [[TMP60:%.*]] = mul i64 [[TMP47]], 3
; CHECK-UF4-NEXT: [[TMP61:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
-; CHECK-UF4-NEXT: [[TMP62:%.*]] = extractelement <vscale x 2 x i64> [[TMP25]], i64 [[TMP61]]
+; CHECK-UF4-NEXT: [[TMP62:%.*]] = extractelement <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], i64 [[TMP61]]
; CHECK-UF4-NEXT: [[TMP63:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], [[TMP60]]
; CHECK-UF4-NEXT: [[TMP64:%.*]] = select i1 [[TMP63]], i64 [[TMP62]], i64 [[TMP59]]
-; CHECK-UF4-NEXT: br label [[FOR_END_LOOPEXIT:%.*]]
-; CHECK-UF4: for.end.loopexit:
-; CHECK-UF4-NEXT: br label [[FOR_END]]
+; CHECK-UF4-NEXT: br label [[FOR_END:%.*]]
; CHECK-UF4: for.end:
-; CHECK-UF4-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP64]], [[FOR_END_LOOPEXIT]] ]
-; CHECK-UF4-NEXT: ret i64 [[RES]]
+; CHECK-UF4-NEXT: ret i64 [[TMP64]]
;
entry:
- %cmp6 = icmp sgt i64 %n, 0
- br i1 %cmp6, label %for.body, label %for.end
+ br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %last = phi i64 [ 0, %entry ], [ %load, %for.body ]
%arrayidx1 = getelementptr inbounds i64, ptr %src, i64 %iv
%load = load i64, ptr %arrayidx1
- %mul = mul i64 %load, 3
%arrayidx2 = getelementptr inbounds i64, ptr %dst, i64 %iv
- store i64 %mul, ptr %arrayidx2
+ store i64 %load, ptr %arrayidx2
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %for.end, label %for.body
for.end:
- %res = phi i64 [0, %entry], [ %last, %for.body]
- ret i64 %res
+ ret i64 %load
}
attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
More information about the llvm-commits
mailing list