[llvm] [LV] Use wide lane masks as the canonical form when tail-folding & interleaving (PR #209484)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 08:26:59 PDT 2026
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/209484
>From b25a57a76cbc08b9e7ab21cbb95cd400e5b6f3dd Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 8 Jul 2026 14:00:32 +0000
Subject: [PATCH 1/9] [LV] Use wide lane masks as the canonical form when
tail-folding & interleaving
Previously, when passing -enable-wide-lane-mask to request the use of wide
lane masks, tryToReplaceALMWithWideALM would attempt to replace several
ActiveLaneMasks created during unrolling with a single mask + extracts.
This PR removes tryToReplaceALMWithWideALM from VPTransforms and makes
wide lane masks the canonical form when interleaving a tail-folded loop
which uses ActiveLaneMask for control flow.
During unrolling, the multiplier is set to UF for ActiveLaneMask and one
ExtractSubvectorForPart instruction for each part is created. Unnecessary
ExtractSubvectorForParts are removed during the generate stage if the loop
is not unrolled, otherwise this creates an extract subvector for each part.
This does not change the default behaviour for tail-folded loops, which
are still not considered for interleaving by default. The decision on
whether to use a single wide active lane mask with extracts or to split
it up into multiple smaller masks is now left to the backend.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 6 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 38 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 111 +------
.../Transforms/Vectorize/VPlanTransforms.h | 1 -
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 25 ++
.../AArch64/fixed-wide-lane-mask.ll | 38 +--
.../AArch64/masked-call-scalarize.ll | 15 +-
.../LoopVectorize/AArch64/masked-call.ll | 79 ++---
.../LoopVectorize/AArch64/pr73894.ll | 8 +-
.../AArch64/sve-tail-folding-unroll.ll | 50 ++-
.../AArch64/sve-wide-lane-mask.ll | 303 +++++++-----------
.../AArch64/uniform-args-call-variants.ll | 134 ++++----
.../LoopVectorize/ARM/active-lane-mask.ll | 2 +-
15 files changed, 367 insertions(+), 476 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5f5767f836948..4e430e47ed659 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -255,11 +255,6 @@ static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"Use predicated EVL instructions for tail folding. If EVL "
"is unsupported, fallback to data-without-lane-mask.")));
-cl::opt<bool> llvm::EnableWideActiveLaneMask(
- "enable-wide-lane-mask", cl::init(false), cl::Hidden,
- cl::desc("Enable use of wide lane masks when used for control flow in "
- "tail-folded loops"));
-
static cl::opt<bool> EnableInterleavedMemAccesses(
"enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
@@ -1214,15 +1209,6 @@ class LoopVectorizationCostModel {
return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
}
- /// Returns true if the use of wide lane masks is requested and the loop is
- /// using tail-folding with a lane mask for control flow.
- bool useWideActiveLaneMask() const {
- if (!EnableWideActiveLaneMask)
- return false;
-
- return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
- }
-
/// Returns true if the instructions in this block requires predication
/// for any reason, e.g. because tail folding now requires a predicate
/// or because the block in the original loop was predicated.
@@ -3644,12 +3630,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// 3. We don't interleave if we think that we will spill registers to memory
// due to the increased register pressure.
- // Only interleave tail-folded loops if wide lane masks are requested, as the
- // overhead of multiple instructions to calculate the predicate is likely
- // not beneficial. If an epilogue is not allowed for any other reason,
- // do not interleave.
- if (!CM.isEpilogueAllowed() &&
- !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()))
+ // Do not interleave tail-folded loops, as the overhead of multiple
+ // instructions to calculate the predicate is likely not beneficial.
+ // If an epilogue is not allowed for any other reason, do not interleave.
+ if (!CM.isEpilogueAllowed() || CM.preferTailFoldedLoop())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index edb24141f6073..d89a001960e09 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1225,15 +1225,18 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
enum {
- FirstOrderRecurrenceSplice =
- Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
- // values of a first-order recurrence.
+ FirstOrderRecurrenceSplice = Instruction::OtherOpsEnd +
+ 1, // Combines the incoming and previous
+ // values of a first-order recurrence.
Not,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
+ // Extracts each unrolled part of a widened active lane mask when used
+ // for control flow.
+ ExtractSubvectorForPart,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 7e7d0f9b6a280..c185d157d23a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -442,6 +442,12 @@ m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
return m_ExtractLastLane(m_ExtractLastPart(Op0));
}
+template <typename Op0_t, typename Op1_t>
+inline VPInstruction_match<VPInstruction::ExtractSubvectorForPart, Op0_t, Op1_t>
+m_ExtractSubvectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(Op0, Op1);
+}
+
template <typename Op0_t>
inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
m_ExtractPenultimateElement(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fc348c42ea7b6..456ad3b2d5380 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -658,6 +658,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::ResumeForEpilogue:
+ case VPInstruction::ExtractSubvectorForPart:
return 2;
case Instruction::InsertElement:
case Instruction::Select:
@@ -793,12 +794,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
- if (State.VF.isScalar())
+ unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ if (State.VF.isScalar() && Multiplier == 1)
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC,
Name);
- ElementCount EC = State.VF.multiplyCoefficientBy(
- cast<VPConstantInt>(getOperand(2))->getZExtValue());
+ ElementCount EC = State.VF.multiplyCoefficientBy(Multiplier);
auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC);
return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
{PredTy, ScalarTC->getType()},
@@ -1104,6 +1105,33 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
+ case VPInstruction::ExtractSubvectorForPart: {
+ Type *DstTy = VectorType::get(getScalarType(), State.VF);
+ unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
+
+ // Return Src if it's an ActiveLaneMask with a multiplier of 1.
+ Value *Src = State.get(getOperand(0));
+ if (Part == 0 && match(getOperand(0),
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_One())))
+ return Src;
+
+ // If the VF is scalar & this is an extract of an active lane mask,
+ // generate an ICMP directly.
+ VPValue *Start, *TC;
+ if (State.VF.isScalar() &&
+ match(getOperand(0),
+ m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
+ Value *StartV = State.get(Start);
+ if (Part > 0)
+ StartV = Builder.CreateAdd(State.get(Start), State.get(getOperand(1)));
+ return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
+ State.get(TC));
+ }
+
+ auto *Idx = ConstantInt::get(Builder.getInt64Ty(),
+ State.VF.getKnownMinValue() * Part);
+ return Builder.CreateExtractVector(DstTy, Src, Idx);
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1628,6 +1656,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLastActive:
+ case VPInstruction::ExtractSubvectorForPart:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -1801,6 +1830,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
+ case VPInstruction::ExtractSubvectorForPart:
+ O << "extract-subvector-for-part";
+ break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b958ca253c7f4..f2796a5d9badf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2035,105 +2035,6 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
-/// Try to replace multiple active lane masks used for control flow with
-/// a single, wide active lane mask instruction followed by multiple
-/// extract subvector intrinsics. This applies to the active lane mask
-/// instructions both in the loop and in the preheader.
-/// Incoming values of all ActiveLaneMaskPHIs are updated to use the
-/// new extracts from the first active lane mask, which has it's last
-/// operand (multiplier) set to UF.
-static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
- unsigned UF) {
- if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1)
- return false;
-
- VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
- auto *Term = &ExitingVPBB->back();
-
- using namespace llvm::VPlanPatternMatch;
- if (!match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue())))))
- return false;
-
- auto *Header = cast<VPBasicBlock>(VectorRegion->getEntry());
- LLVMContext &Ctx = Plan.getContext();
-
- auto ExtractFromALM = [&](VPInstruction *ALM,
- SmallVectorImpl<VPValue *> &Extracts) {
- DebugLoc DL = ALM->getDebugLoc();
- for (unsigned Part = 0; Part < UF; ++Part) {
- SmallVector<VPValue *> Ops;
- Ops.append({ALM, Plan.getConstantInt(64, VF.getKnownMinValue() * Part)});
- auto *Ext =
- new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops,
- IntegerType::getInt1Ty(Ctx), {}, {}, DL);
- Extracts[Part] = Ext;
- Ext->insertAfter(ALM);
- }
- };
-
- // Create a list of each active lane mask phi, ordered by unroll part.
- SmallVector<VPActiveLaneMaskPHIRecipe *> Phis(UF, nullptr);
- for (VPRecipeBase &R : Header->phis()) {
- auto *Phi = dyn_cast<VPActiveLaneMaskPHIRecipe>(&R);
- if (!Phi)
- continue;
- VPValue *Index = nullptr;
- match(Phi->getBackedgeValue(),
- m_ActiveLaneMask(m_VPValue(Index), m_VPValue(), m_VPValue()));
- assert(Index && "Expected index from ActiveLaneMask instruction");
-
- uint64_t Part;
- if (match(Index,
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
- m_VPValue(), m_Mul(m_VPValue(), m_ConstantInt(Part)))))
- Phis[Part] = Phi;
- else {
- // Anything other than a CanonicalIVIncrementForPart is part 0
- assert(!match(
- Index,
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()));
- Phis[0] = Phi;
- }
- }
-
- assert(all_of(Phis, not_equal_to(nullptr)) &&
- "Expected one VPActiveLaneMaskPHIRecipe for each unroll part");
-
- auto *EntryALM = cast<VPInstruction>(Phis[0]->getStartValue());
- auto *LoopALM = cast<VPInstruction>(Phis[0]->getBackedgeValue());
-
- assert((EntryALM->getOpcode() == VPInstruction::ActiveLaneMask &&
- LoopALM->getOpcode() == VPInstruction::ActiveLaneMask) &&
- "Expected incoming values of Phi to be ActiveLaneMasks");
-
- // When using wide lane masks, the return type of the get.active.lane.mask
- // intrinsic is VF x UF (last operand).
- VPValue *ALMMultiplier = Plan.getConstantInt(64, UF);
- EntryALM->setOperand(2, ALMMultiplier);
- LoopALM->setOperand(2, ALMMultiplier);
-
- // Create UF x extract vectors and insert into preheader.
- SmallVector<VPValue *> EntryExtracts(UF);
- ExtractFromALM(EntryALM, EntryExtracts);
-
- // Create UF x extract vectors and insert before the loop compare & branch,
- // updating the compare to use the first extract.
- SmallVector<VPValue *> LoopExtracts(UF);
- ExtractFromALM(LoopALM, LoopExtracts);
- VPInstruction *Not = cast<VPInstruction>(Term->getOperand(0));
- Not->setOperand(0, LoopExtracts[0]);
-
- // Update the incoming values of active lane mask phis.
- for (unsigned Part = 0; Part < UF; ++Part) {
- Phis[Part]->setStartValue(EntryExtracts[Part]);
- Phis[Part]->setBackedgeValue(LoopExtracts[Part]);
- }
-
- return true;
-}
-
/// Try to simplify the branch condition of \p Plan. This may restrict the
/// resulting plan to \p BestVF and \p BestUF.
static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
@@ -2150,9 +2051,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
m_CombineOr(m_CanIVInc, m_c_Add(m_CanIVInc, m_LiveIn())),
m_VPValue())) ||
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue()))))) {
+ m_VPValue(), m_VPValue(), m_VPValue())))) ||
+ match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount or BranchOnCond(Not(ActiveLaneMask)).
+ // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask))
+ // or BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMask), 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
@@ -2196,8 +2101,8 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF);
- MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
+ bool MadeChange =
+ simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
if (MadeChange) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 4b59d37150ff4..7d2a263efce6d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -39,7 +39,6 @@ class VPRecipeBuilder;
struct VFRange;
LLVM_ABI_FOR_TEST extern cl::opt<bool> VerifyEachVPlan;
-LLVM_ABI_FOR_TEST extern cl::opt<bool> EnableWideActiveLaneMask;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_ABI_FOR_TEST extern cl::opt<bool> VPlanPrintBeforeAll;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a430de94cca14..4c15afc0466e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -471,6 +471,31 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
+ if (match(&R,
+ m_ActiveLaneMask(m_VPValue(), m_VPValue(Op1), m_VPValue(Op2)))) {
+ auto *ALM = cast<VPInstruction>(&R);
+ auto *User = ALM->getSingleUser();
+ // Widen ActiveLaneMask when used for control flow.
+ if (User && match(User, m_ExtractSubvectorForPart(m_Specific(ALM),
+ m_ZeroInt()))) {
+ addUniformForAllParts(ALM);
+ ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ continue;
+ }
+ }
+
+ if (match(&R, m_ExtractSubvectorForPart(m_VPValue(Op0), m_VPValue(Op1)))) {
+ VPRecipeBase *InsertPt = &R;
+ for (unsigned Part = 1; Part != UF; ++Part) {
+ auto *Ext = new VPInstruction(VPInstruction::ExtractSubvectorForPart,
+ {Op0, Plan.getConstantInt(64, Part)});
+ Ext->insertAfter(InsertPt);
+ addRecipeForPart(&R, Ext, Part);
+ InsertPt = Ext;
+ }
+ continue;
+ }
+
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {
addUniformForAllParts(SingleDef);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index 17cc76c52c7e3..bff2a61711124 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
target triple = "aarch64-unknown-linux"
@@ -10,13 +10,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask(
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: br label [[ENTRY:%.*]]
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
-; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[TMP3:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -32,20 +32,20 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: br label [[ENTRY:%.*]]
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
-; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
; CHECK-UF4-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: br label [[VECTOR_BODY1:%.*]]
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
-; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[ENTRY]] ], [ [[TMP9:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[ENTRY]] ], [ [[TMP10:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[ENTRY]] ], [ [[TMP12:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-UF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-UF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -59,13 +59,13 @@ define void @fixed_wide_active_lane_mask(ptr noalias %dst, ptr noalias readonly
; CHECK-UF4-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP19]], <4 x i1> [[ACTIVE_LANE_MASK6]])
; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
-; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
-; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
; CHECK-UF4-NEXT: [[TMP9]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP10]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP11]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-UF4-NEXT: [[TMP12]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
; CHECK-UF4-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
; CHECK-UF4-NEXT: [[TMP20:%.*]] = xor i1 [[TMP21]], true
-; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF4: middle.block:
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index d7c4641429480..9292b8bb2c43d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -101,13 +101,14 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT9:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[PRED_STORE_CONTINUE8]] ]
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
@@ -144,9 +145,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE8]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE8]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT9]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP13]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 018c280b693d8..92bd724aeb415 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -82,13 +82,14 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -100,9 +101,9 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -215,13 +216,14 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -239,9 +241,9 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP13]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -373,13 +375,14 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP19]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP23]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -403,9 +406,9 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP22]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -680,13 +683,14 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -698,9 +702,9 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
; TFA_INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -855,15 +859,16 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP21]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
@@ -884,9 +889,9 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> splat (double -0.000000e+00)
; TFA_INTERLEAVE-NEXT: [[TMP16]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP14]], <vscale x 2 x double> [[TMP15]])
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[TMP20]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -991,6 +996,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1013,8 +1019,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index 3e61a933d0ecb..a01923d1ce361 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -12,6 +12,7 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -30,21 +31,22 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK: pred.load.continue:
; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
; CHECK-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5]]
-; CHECK: pred.load.if4:
+; CHECK: pred.load.if3:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]]
-; CHECK: pred.load.continue5:
+; CHECK: pred.load.continue4:
; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF4]] ]
; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP8]], i32 [[VEC_PHI]])
; CHECK-NEXT: [[TMP16:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI3]])
; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]]
; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
; CHECK-NEXT: [[TMP20:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 83d9a717d04d2..55b7b14c6139e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -13,12 +13,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 2
; CHECK-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 12)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -29,6 +28,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
+; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]]
; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]]
@@ -37,13 +38,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP57]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK8]])
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP60]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK9]])
; CHECK-NEXT: [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]]
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP1]]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP28]]
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX_NEXT10]], [[TMP30]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP9]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP10]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP11]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT10]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 12)
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP36:%.*]] = xor i1 [[TMP35]], true
; CHECK-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -76,12 +75,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP5]], 2
; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NEXT: [[TMP28:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP28]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP30]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY5:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_ENTRY1]], i64 12)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -92,6 +90,8 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[COND_PTR:%.*]], i64 [[INDEX6]]
+; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP28]]
; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP30]]
@@ -116,13 +116,11 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP79]], <vscale x 4 x i1> [[TMP71]])
; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP82]], <vscale x 4 x i1> [[TMP72]])
; CHECK-NEXT: [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP6]]
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP1]]
-; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP28]]
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[INDEX_NEXT13]], [[TMP30]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP21]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP22]], i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP23]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT13]], i64 [[UMAX]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 4)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 12)
; CHECK-NEXT: [[TMP66:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP67:%.*]] = xor i1 [[TMP66]], true
; CHECK-NEXT: br i1 [[TMP67]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 2e6e3a41addc5..f77faf7eac54e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
-; RUN: opt -S --passes=loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix CHECK-TF
-; RUN: opt -S --passes=forceattrs,loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
+; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
target triple = "aarch64-unknown-linux"
@@ -10,115 +9,80 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-UF1-LABEL: define void @scalable_wide_active_lane_mask(
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF1-NEXT: entry:
-; CHECK-UF1-NEXT: br label [[VECTOR_PH1:%.*]]
+; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF1-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
-; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH1]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-UF1-NEXT: [[TMP6:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-UF1-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP6]], ptr align 1 [[TMP13]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP18]]
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP3]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP4]], ptr align 1 [[TMP5]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP14:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-UF1-NEXT: [[TMP11:%.*]] = xor i1 [[TMP14]], true
-; CHECK-UF1-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF1: middle.block:
;
; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF4-NEXT: entry:
-; CHECK-UF4-NEXT: br label [[VECTOR_PH1:%.*]]
+; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
-; CHECK-UF4-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP61]], 4
-; CHECK-UF4-NEXT: [[TMP62:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
-; CHECK-UF4-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
-; CHECK-UF4-NEXT: [[TMP17:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-UF4-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
-; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP16]], [[VECTOR_PH1]] ], [ [[TMP55:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 16 x i1> [ [[TMP17]], [[VECTOR_PH1]] ], [ [[TMP56:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 16 x i1> [ [[TMP18]], [[VECTOR_PH1]] ], [ [[TMP57:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], [[VECTOR_PH1]] ], [ [[TMP58:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP32:%.*]] = shl nuw nsw i64 [[TMP1]], 1
-; CHECK-UF4-NEXT: [[TMP29:%.*]] = mul nuw nsw i64 [[TMP1]], 3
-; CHECK-UF4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP32]]
-; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 [[TMP29]]
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP20]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP24]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP33]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP30]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 16 x i8> poison)
-; CHECK-UF4-NEXT: [[TMP25:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP26:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD9]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP27:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD10]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP28:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD11]], splat (i8 3)
-; CHECK-UF4-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP32]]
-; CHECK-UF4-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[TMP35]], i64 [[TMP29]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP25]], ptr align 1 [[TMP35]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP26]], ptr align 1 [[TMP39]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK6]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP27]], ptr align 1 [[TMP42]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK7]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP28]], ptr align 1 [[TMP45]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK8]])
-; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP62]]
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 16 x i8> poison)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]], splat (i8 3)
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP17]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP17]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP15]], ptr align 1 [[TMP19]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP16]], ptr align 1 [[TMP20]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP58]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 48)
-; CHECK-UF4-NEXT: [[TMP57]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 32)
-; CHECK-UF4-NEXT: [[TMP56]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
-; CHECK-UF4-NEXT: [[TMP55]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-UF4-NEXT: [[TMP59:%.*]] = extractelement <vscale x 16 x i1> [[TMP55]], i64 0
-; CHECK-UF4-NEXT: [[TMP60:%.*]] = xor i1 [[TMP59]], true
-; CHECK-UF4-NEXT: br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4-NEXT: [[TMP21]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP22]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
+; CHECK-UF4-NEXT: [[TMP23]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 32)
+; CHECK-UF4-NEXT: [[TMP24]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 48)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP21]], i64 0
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true
+; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-UF4: middle.block:
;
-; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask(
-; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-TF-NEXT: entry:
-; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-TF: vector.ph:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-TF: vector.body:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP10]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 16 x i8> poison)
-; CHECK-TF-NEXT: [[TMP13:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
-; CHECK-TF-NEXT: [[TMP14:%.*]] = mul <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]], splat (i8 3)
-; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i64 [[TMP10]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP13]], ptr align 1 [[TMP15]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP14]], ptr align 1 [[TMP18]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK1]])
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP19]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
-; CHECK-TF-NEXT: [[TMP20]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 16 x i1> [[TMP20]], i64 0
-; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
-; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-TF: middle.block:
-;
entry:
br label %for.body
@@ -142,123 +106,85 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl
; CHECK-UF1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF1-NEXT: entry:
; CHECK-UF1-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-UF1-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
; CHECK-UF1: for.body.preheader:
; CHECK-UF1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF1-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.body:
; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-UF1-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP3]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP4]], ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF1-NEXT: [[TMP7:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP7]], true
-; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF1: middle.block:
;
; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double(
; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF4-NEXT: entry:
; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
; CHECK-UF4: for.body.preheader:
; CHECK-UF4-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-UF4: vector.ph:
-; CHECK-UF4-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-UF4-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP14:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
-; CHECK-UF4-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
-; CHECK-UF4-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-UF4-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP11]], [[VECTOR_PH]] ], [ [[TMP50:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 2 x i1> [ [[TMP12]], [[VECTOR_PH]] ], [ [[TMP51:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], [[VECTOR_PH]] ], [ [[TMP52:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 2 x i1> [ [[TMP14]], [[VECTOR_PH]] ], [ [[TMP53:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP21:%.*]] = shl nuw nsw i64 [[TMP1]], 1
-; CHECK-UF4-NEXT: [[TMP24:%.*]] = mul nuw nsw i64 [[TMP1]], 3
-; CHECK-UF4-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP22:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP21]]
-; CHECK-UF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP24]]
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP15]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP29]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP22]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP25]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 2 x double> poison)
-; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP17:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD9]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP18:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD10]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP19:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD11]], splat (double 3.000000e+00)
-; CHECK-UF4-NEXT: [[TMP30:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-UF4-NEXT: [[TMP34:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP1]]
-; CHECK-UF4-NEXT: [[TMP37:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP21]]
-; CHECK-UF4-NEXT: [[TMP40:%.*]] = getelementptr inbounds double, ptr [[TMP30]], i64 [[TMP24]]
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP30]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP17]], ptr align 8 [[TMP34]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP18]], ptr align 8 [[TMP37]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK7]])
-; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP19]], ptr align 8 [[TMP40]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK8]])
-; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP11]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK3]], <vscale x 2 x double> poison)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD4]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD5]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP1]]
+; CHECK-UF4-NEXT: [[TMP19:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP17]], i64 [[TMP9]]
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP15]], ptr align 8 [[TMP19]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP16]], ptr align 8 [[TMP20]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-UF4-NEXT: [[TMP53]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
-; CHECK-UF4-NEXT: [[TMP52]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
-; CHECK-UF4-NEXT: [[TMP51]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; CHECK-UF4-NEXT: [[TMP50]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-UF4-NEXT: [[TMP54:%.*]] = extractelement <vscale x 2 x i1> [[TMP50]], i64 0
-; CHECK-UF4-NEXT: [[TMP55:%.*]] = xor i1 [[TMP54]], true
-; CHECK-UF4-NEXT: br i1 [[TMP55]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF4-NEXT: [[TMP21]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP22]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-UF4-NEXT: [[TMP23]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP24]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 6)
+; CHECK-UF4-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP21]], i64 0
+; CHECK-UF4-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true
+; CHECK-UF4-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-UF4: middle.block:
;
-; CHECK-TF-LABEL: define void @scalable_wide_active_lane_mask_double(
-; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-TF-NEXT: entry:
-; CHECK-TF-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
-; CHECK-TF-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
-; CHECK-TF: for.body.preheader:
-; CHECK-TF-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK-TF: vector.ph:
-; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 1
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-TF-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-TF-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK-TF: vector.body:
-; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP10]]
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP12]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; CHECK-TF-NEXT: [[TMP13:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
-; CHECK-TF-NEXT: [[TMP14:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD2]], splat (double 3.000000e+00)
-; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds double, ptr [[TMP15]], i64 [[TMP10]]
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP13]], ptr align 8 [[TMP15]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP14]], ptr align 8 [[TMP18]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
-; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-TF-NEXT: [[TMP19]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; CHECK-TF-NEXT: [[TMP20]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 2 x i1> [[TMP20]], i64 0
-; CHECK-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
-; CHECK-TF-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-TF: middle.block:
-;
entry:
%cmp6 = icmp sgt i64 %n, 0
br i1 %cmp6, label %for.body, label %for.end
@@ -279,4 +205,3 @@ for.end:
}
attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
-
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index 094ce721f20c6..0a5a00af05e9d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -42,30 +42,31 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -125,30 +126,31 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; INTERLEAVE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -196,38 +198,40 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: entry:
; INTERLEAVE-NEXT: br label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[N]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[N]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]]
+; INTERLEAVE-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]]
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
-; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[PRED_STORE_CONTINUE4]] ]
+; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE3]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi i1 [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE3]] ]
; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; INTERLEAVE: pred.store.if:
-; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[TMP0]], align 8
-; INTERLEAVE-NEXT: [[TMP2:%.*]] = call double @foo(double [[TMP1]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8
+; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = call double @foo(double [[TMP3]], i64 [[INDEX]]) #[[ATTR4:[0-9]+]]
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]]
; INTERLEAVE: pred.store.continue:
-; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; INTERLEAVE: pred.store.if3:
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP4]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = load double, ptr [[TMP5]], align 8
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call double @foo(double [[TMP6]], i64 [[TMP4]]) #[[ATTR4]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP4]]
-; INTERLEAVE-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
-; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; INTERLEAVE: pred.store.continue4:
+; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; INTERLEAVE: pred.store.if2:
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP6]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP7]], align 8
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = call double @foo(double [[TMP8]], i64 [[TMP6]]) #[[ATTR4]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP6]]
+; INTERLEAVE-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8
+; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE3]]
+; INTERLEAVE: pred.store.continue3:
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], 1
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP9]], [[N]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
-; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; INTERLEAVE-NEXT: [[TMP11]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i64 [[INDEX_NEXT]], 1
+; INTERLEAVE-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[N]]
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP11]], true
+; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
index 6f82b9301fb45..9a8e204b53e8c 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -S | FileCheck %s
target triple = "thumbv8.1m.main-arm-unknown-eabihf"
>From 4f05c7744f821861d77902701c2ac3e89036935e Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 14 Jul 2026 14:23:08 +0000
Subject: [PATCH 2/9] - Simplify ExtractSubvectorForPart recipe for removing
unnecessary extract - Move ExtractSubvectorForPart unrolling to
unrollRecipeByUF - Fix test failures
---
llvm/lib/Transforms/Vectorize/VPlan.h | 6 +++---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +++-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 18 ++++++------------
.../VPlan/AArch64/sve-tail-folding-forced.ll | 6 ++++--
.../VPlan/AArch64/vplan-printing-alias-mask.ll | 1 +
.../VPlan/AArch64/vplan-printing.ll | 6 ++++--
6 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d89a001960e09..93ff9c6d2f78f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1225,9 +1225,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public:
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
enum {
- FirstOrderRecurrenceSplice = Instruction::OtherOpsEnd +
- 1, // Combines the incoming and previous
- // values of a first-order recurrence.
+ FirstOrderRecurrenceSplice =
+ Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
+ // values of a first-order recurrence.
Not,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 456ad3b2d5380..7f4f88ad88c8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1106,13 +1106,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
case VPInstruction::ExtractSubvectorForPart: {
+ Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
- // Return Src if it's an ActiveLaneMask with a multiplier of 1.
- Value *Src = State.get(getOperand(0));
- if (Part == 0 && match(getOperand(0),
- m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_One())))
+ if (Src->getType() == DstTy)
return Src;
// If the VF is scalar & this is an extract of an active lane mask,
@@ -1123,7 +1121,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
- StartV = Builder.CreateAdd(State.get(Start), State.get(getOperand(1)));
+ StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
State.get(TC));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 4c15afc0466e2..2e08a5d88a2b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -332,6 +332,12 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
+ if (match(&R, m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(
+ m_VPValue(Op), m_VPValue()))) {
+ Copy->setOperand(0, Op);
+ Copy->setOperand(1, Plan.getConstantInt(64, Part));
+ continue;
+ }
if (isa<VPVectorPointerRecipe, VPWidenCanonicalIVRecipe>(R)) {
VPBuilder Builder(&R);
const DataLayout &DL = Plan.getDataLayout();
@@ -484,18 +490,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
}
}
- if (match(&R, m_ExtractSubvectorForPart(m_VPValue(Op0), m_VPValue(Op1)))) {
- VPRecipeBase *InsertPt = &R;
- for (unsigned Part = 1; Part != UF; ++Part) {
- auto *Ext = new VPInstruction(VPInstruction::ExtractSubvectorForPart,
- {Op0, Plan.getConstantInt(64, Part)});
- Ext->insertAfter(InsertPt);
- addRecipeForPart(&R, Ext, Part);
- InsertPt = Ext;
- }
- continue;
- }
-
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
if (SingleDef && vputils::isUniformAcrossVFsAndUFs(SingleDef)) {
addUniformForAllParts(SingleDef);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index c01da392c49ee..6305b642085a2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -20,20 +20,22 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
; VPLANS-NEXT: vp<[[INDV:%[0-9]+]]> = CANONICAL-IV
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.body:
-; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%.+]]>
+; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY_EXTRACT]]>, vp<[[LANEMASK_LOOP:%.+]]>
; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1>, vp<[[VF]]>
; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]>
; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep>
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
; VPLANS-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index c257a5a14cc27..a208e8fe477a0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -26,6 +26,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 33b44d49f6be7..89b76a5529f08 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -138,13 +138,14 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vp<[[CAN_IV:%.+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%[0-9]+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
@@ -157,7 +158,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
>From 71c173ef1ce97df5e8f8fa17b8f5c903e81627a5 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 10:01:55 +0000
Subject: [PATCH 3/9] - Rebase & update vplan-printing-alias-mask.ll
---
.../LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index a208e8fe477a0..97868414a8d70 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -31,7 +31,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
; FINAL-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; FINAL-NEXT: ACTIVE-LANE-MASK-PHI vp<[[VP8:%[0-9]+]]> = phi vp<%extract.entry.alm.part>, vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = and vp<[[VP8]]>, vp<[[VP4]]>
; FINAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, vp<%index>
; FINAL-NEXT: WIDEN ir<%ld.a> = load ir<%ptr.a>, vp<[[VP9]]>
@@ -42,7 +42,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%active.lane.mask.next>
+; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
; FINAL-EMPTY:
>From 9ce79c106470dc41ae5eaf201cbce4c656653413 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 17 Jul 2026 13:16:29 +0000
Subject: [PATCH 4/9] - Add ActiveLaneMaskForControlFlow - Remove Op1/Op2
---
llvm/lib/Transforms/Vectorize/VPlan.h | 1 +
.../lib/Transforms/Vectorize/VPlanPatternMatch.h | 9 +++++++++
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 +++++++++++++---
.../lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 15 +++++----------
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 8 ++++++--
.../VPlan/AArch64/sve-tail-folding-forced.ll | 4 ++--
.../VPlan/AArch64/vplan-printing-alias-mask.ll | 4 ++--
.../VPlan/AArch64/vplan-printing.ll | 4 ++--
9 files changed, 45 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 93ff9c6d2f78f..39ec0e5a2f09f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1234,6 +1234,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
+ ActiveLaneMaskForControlFlow,
// Extracts each unrolled part of a widened active lane mask when used
// for control flow.
ExtractSubvectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index c185d157d23a5..9eccf491ac104 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -460,6 +460,15 @@ m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
}
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPInstruction_match<VPInstruction::ActiveLaneMaskForControlFlow, Op0_t,
+ Op1_t, Op2_t>
+m_ActiveLaneMaskForControlFlow(const Op0_t &Op0, const Op1_t &Op1,
+ const Op2_t &Op2) {
+ return m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>(Op0, Op1,
+ Op2);
+}
+
inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
return m_VPInstruction<VPInstruction::AnyOf>();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7f4f88ad88c8b..8f34890c38c70 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -497,6 +497,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
assert(Op0Ty->isIntegerTy() && "expected integer operand");
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
@@ -663,6 +664,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::InsertElement:
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::ReductionStartVector:
return 3;
case Instruction::Call:
@@ -786,7 +788,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
Name);
}
- case VPInstruction::ActiveLaneMask: {
+ case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow: {
// Get first lane of vector induction variable.
Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
// Get the original loop tripcount.
@@ -1118,7 +1121,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
VPValue *Start, *TC;
if (State.VF.isScalar() &&
match(getOperand(0),
- m_ActiveLaneMask(m_VPValue(Start), m_VPValue(TC), m_VPValue()))) {
+ m_ActiveLaneMaskForControlFlow(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
@@ -1437,7 +1441,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
TargetTransformInfo::SK_Splice, cast<VectorType>(VectorTy),
cast<VectorType>(VectorTy), {}, Ctx.CostKind, -1);
}
- case VPInstruction::ActiveLaneMask: {
+ case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow: {
Type *ArgTy = getOperand(0)->getScalarType();
unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
@@ -1648,6 +1653,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::IncomingAliasMask:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
@@ -1705,6 +1711,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return vputils::onlyFirstLaneUsed(this);
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
+ case VPInstruction::ActiveLaneMaskForControlFlow:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -1777,6 +1784,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ActiveLaneMask:
O << "active lane mask";
break;
+ case VPInstruction::ActiveLaneMaskForControlFlow:
+ O << "active lane mask control flow";
+ break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f2796a5d9badf..505b1c35c9857 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2053,11 +2053,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
m_VPValue(), m_VPValue(), m_VPValue())))) ||
match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
- m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue()),
m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask))
- // or BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMask), 0))
+ // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
+ // BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMaskForControlFlow),
+ // 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 2e08a5d88a2b2..45fdc1180ac50 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -477,17 +477,12 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
- if (match(&R,
- m_ActiveLaneMask(m_VPValue(), m_VPValue(Op1), m_VPValue(Op2)))) {
+ if (match(&R, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
- auto *User = ALM->getSingleUser();
- // Widen ActiveLaneMask when used for control flow.
- if (User && match(User, m_ExtractSubvectorForPart(m_Specific(ALM),
- m_ZeroInt()))) {
- addUniformForAllParts(ALM);
- ALM->setOperand(2, Plan.getConstantInt(64, UF));
- continue;
- }
+ addUniformForAllParts(ALM);
+ ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ continue;
}
auto *SingleDef = dyn_cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index a4d6cc0ecb480..52d753c460950 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -196,10 +196,14 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
// loop region's abstract header mask; afterwards it is an active lane mask
// (an intrinsic or a phi), or the icmp checked below.
if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
- match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()))
+ match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()) ||
+ match(Mask,
+ m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>()))
continue;
- if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
+ if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
+ match(Mask, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
+ m_VPValue())))
continue;
CmpPredicate Pred;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 6305b642085a2..60fd0afc8c02f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -19,7 +19,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask control flow vp<[[VF_PER_PART]]>, vp<[[TC]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
@@ -34,7 +34,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask control flow vp<[[INC]]>, vp<[[TC]]>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 97868414a8d70..3226ed2ba779c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -25,7 +25,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
-; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow ir<0>, ir<%n>, ir<1>
; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
@@ -41,7 +41,7 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<%index.next>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<%index.next>, ir<%n>, ir<1>
; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 89b76a5529f08..9da97a2276b6a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -137,7 +137,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow vp<%index.part.next>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
@@ -157,7 +157,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<[[PART_IDX]]>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
>From 4b0ee0ca67034b89c8fe9d8e20879a0309dfbd40 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 20 Jul 2026 15:59:00 +0000
Subject: [PATCH 5/9] - Rename ActiveLaneMaskForControlFlow ->
WideActiveLaneMask - Rename ExtractSubvectorForPart -> ExtractVectorForPart -
Update comments in VPTransforms to reflect wide ALM + extracts
---
llvm/lib/Transforms/Vectorize/VPlan.h | 7 ++--
.../Transforms/Vectorize/VPlanLowering.cpp | 36 ++++++++++++-------
.../Transforms/Vectorize/VPlanPatternMatch.h | 16 ++++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 30 ++++++++--------
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 +++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 6 ++--
.../Transforms/Vectorize/VPlanVerifier.cpp | 7 ++--
.../VPlan/AArch64/sve-tail-folding-forced.ll | 8 ++---
.../AArch64/vplan-printing-alias-mask.ll | 8 ++---
.../VPlan/AArch64/vplan-printing.ll | 8 ++---
10 files changed, 71 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 39ec0e5a2f09f..9f8e4eb3c44b1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1234,10 +1234,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// mask[i] = icmpt ult (op0 + i), op1
// The size of the mask returned is VF * Multiplier (UF, third op).
ActiveLaneMask,
- ActiveLaneMaskForControlFlow,
- // Extracts each unrolled part of a widened active lane mask when used
- // for control flow.
- ExtractSubvectorForPart,
+ WideActiveLaneMask,
+ // Extracts each unrolled part of a (VF * UF) widened vector/mask.
+ ExtractVectorForPart,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 3a64567dfab0b..e3b0ecc37c1dc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -117,7 +117,7 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
// the loop terminator with a branch-on-cond recipe with the negated
-// active-lane-mask as operand. Note that this turns the loop into an
+// wide-active-lane-mask as operand. Note that this turns the loop into an
// uncountable one. Only the existing terminator is replaced, all other existing
// recipes/users remain unchanged, except for poison-generating flags being
// dropped from the canonical IV increment. Return the created
@@ -127,15 +127,18 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
//
// vector.ph:
// %EntryInc = canonical-iv-increment-for-part CanonicalIVStart
-// %EntryALM = active-lane-mask %EntryInc, TC
+// %EntryALM = wide-active-lane-mask %EntryInc, TC
+// %EntryALMPart = extract-vector-for-part %EntryALM, ir<0>
//
// vector.body:
// ...
-// %P = active-lane-mask-phi [ %EntryALM, %vector.ph ], [ %ALM, %vector.body ]
+// %P = active-lane-mask-phi [ %EntryALMPart, %vector.ph ],
+// [ %ALMPart, %vector.body ]
// ...
// %InLoopInc = canonical-iv-increment-for-part CanonicalIVIncrement
-// %ALM = active-lane-mask %InLoopInc, TC
-// %Negated = Not %ALM
+// %ALM = wide-active-lane-mask %InLoopInc, TC
+// %ALMPart = extract-vector-for-part %ALM, ir<0>
+// %Negated = Not %ALMPart
// branch-on-cond %Negated
//
static VPActiveLaneMaskPHIRecipe *
@@ -147,13 +150,14 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
// TODO: Check if dropping the flags is needed.
TopRegion->clearCanonicalIVNUW(CanonicalIVIncrement);
DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
- // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
- // we have to take unrolling into account. Each part needs to start at
- // Part * VF
+ // We can't use StartV directly in the WideActiveLaneMask
+ // VPInstruction, since we have to take unrolling into account.
+ // Each part needs to start at Part * VF
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
- // Create the ActiveLaneMask instruction using the correct start values.
+ // Create the WideActiveLaneMask instruction using the correct
+ // start values.
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
@@ -161,15 +165,18 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
{StartV, VF}, {}, DL, "index.part.next");
- // Create the active lane mask instruction in the VPlan preheader.
+ // Create the wide active lane mask instruction in the VPlan preheader.
VPValue *ALMMultiplier =
Plan.getConstantInt(TopRegion->getCanonicalIVType(), 1);
- auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ auto *EntryALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
{EntryIncrement, TC, ALMMultiplier}, DL,
"active.lane.mask.entry");
+ EntryALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {EntryALM, Plan.getConstantInt(64, 0)}, DL,
+ "extract.entry.alm.part");
// Now create the ActiveLaneMaskPhi recipe in the main loop using the
- // preheader ActiveLaneMask instruction.
+ // preheader WideActiveLaneMask instruction.
auto *LaneMaskPhi =
new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc::getUnknown());
auto *HeaderVPBB = TopRegion->getEntryBasicBlock();
@@ -182,9 +189,12 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
auto *InLoopIncrement = Builder.createOverflowingOp(
VPInstruction::CanonicalIVIncrementForPart,
{CanonicalIVIncrement, &Plan.getVF()}, {}, DL);
- auto *ALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ auto *ALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
{InLoopIncrement, TC, ALMMultiplier}, DL,
"active.lane.mask.next");
+ ALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
+ {ALM, Plan.getConstantInt(64, 0)}, DL,
+ "extract.next.alm.part");
LaneMaskPhi->addBackedgeValue(ALM);
// Replace the original terminator with BranchOnCond. We have to invert the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 9eccf491ac104..1fe1715dc35e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -443,9 +443,9 @@ m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
}
template <typename Op0_t, typename Op1_t>
-inline VPInstruction_match<VPInstruction::ExtractSubvectorForPart, Op0_t, Op1_t>
-m_ExtractSubvectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
- return m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(Op0, Op1);
+inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op0_t, Op1_t>
+m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op0, Op1);
}
template <typename Op0_t>
@@ -461,12 +461,10 @@ m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
}
template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ActiveLaneMaskForControlFlow, Op0_t,
- Op1_t, Op2_t>
-m_ActiveLaneMaskForControlFlow(const Op0_t &Op0, const Op1_t &Op1,
- const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>(Op0, Op1,
- Op2);
+inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
+ Op2_t>
+m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+ return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op0, Op1, Op2);
}
inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8f34890c38c70..149db867d1c5e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -497,7 +497,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
assert(Op0Ty->isIntegerTy() && "expected integer operand");
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
@@ -659,12 +659,12 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::ResumeForEpilogue:
- case VPInstruction::ExtractSubvectorForPart:
+ case VPInstruction::ExtractVectorForPart:
return 2;
case Instruction::InsertElement:
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
return 3;
case Instruction::Call:
@@ -789,7 +789,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Name);
}
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow: {
+ case VPInstruction::WideActiveLaneMask: {
// Get first lane of vector induction variable.
Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
// Get the original loop tripcount.
@@ -1108,7 +1108,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
- case VPInstruction::ExtractSubvectorForPart: {
+ case VPInstruction::ExtractVectorForPart: {
Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
@@ -1121,8 +1121,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
VPValue *Start, *TC;
if (State.VF.isScalar() &&
match(getOperand(0),
- m_ActiveLaneMaskForControlFlow(m_VPValue(Start), m_VPValue(TC),
- m_VPValue()))) {
+ m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()))) {
Value *StartV = State.get(Start);
if (Part > 0)
StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
@@ -1442,7 +1442,7 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
cast<VectorType>(VectorTy), {}, Ctx.CostKind, -1);
}
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow: {
+ case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
@@ -1653,14 +1653,14 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLastActive:
- case VPInstruction::ExtractSubvectorForPart:
+ case VPInstruction::ExtractVectorForPart:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -1711,7 +1711,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return vputils::onlyFirstLaneUsed(this);
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
- case VPInstruction::ActiveLaneMaskForControlFlow:
+ case VPInstruction::WideActiveLaneMask:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -1784,8 +1784,8 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ActiveLaneMask:
O << "active lane mask";
break;
- case VPInstruction::ActiveLaneMaskForControlFlow:
- O << "active lane mask control flow";
+ case VPInstruction::WideActiveLaneMask:
+ O << "wide active lane mask";
break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
@@ -1838,8 +1838,8 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
- case VPInstruction::ExtractSubvectorForPart:
- O << "extract-subvector-for-part";
+ case VPInstruction::ExtractVectorForPart:
+ O << "extract-vector-for-part";
break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 505b1c35c9857..8232957864c8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2052,13 +2052,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
m_VPValue())) ||
match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
m_VPValue(), m_VPValue(), m_VPValue())))) ||
- match(Term, m_BranchOnCond(m_Not(m_ExtractSubvectorForPart(
- m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue()),
- m_ZeroInt()))))) {
+ match(Term,
+ m_BranchOnCond(m_Not(m_ExtractVectorForPart(
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
+ m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
// latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
- // BranchOnCond(Not(ExtractSubvectorForPart(ActiveLaneMaskForControlFlow),
+ // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask),
// 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 45fdc1180ac50..f3d3435962d8e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -332,7 +332,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
- if (match(&R, m_VPInstruction<VPInstruction::ExtractSubvectorForPart>(
+ if (match(&R, m_VPInstruction<VPInstruction::ExtractVectorForPart>(
m_VPValue(Op), m_VPValue()))) {
Copy->setOperand(0, Op);
Copy->setOperand(1, Plan.getConstantInt(64, Part));
@@ -477,8 +477,8 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
continue;
}
- if (match(&R, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue()))) {
+ if (match(&R,
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
addUniformForAllParts(ALM);
ALM->setOperand(2, Plan.getConstantInt(64, UF));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 52d753c460950..68f4eb44d7ff5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -197,13 +197,12 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
// (an intrinsic or a phi), or the icmp checked below.
if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()) ||
- match(Mask,
- m_VPInstruction<VPInstruction::ActiveLaneMaskForControlFlow>()))
+ match(Mask, m_VPInstruction<VPInstruction::WideActiveLaneMask>()))
continue;
if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
- match(Mask, m_ActiveLaneMaskForControlFlow(m_VPValue(), m_VPValue(),
- m_VPValue())))
+ match(Mask,
+ m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
continue;
CmpPredicate Pred;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 60fd0afc8c02f..86ea9eb5a2e9e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -19,8 +19,8 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = active lane mask control flow vp<[[VF_PER_PART]]>, vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-subvector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-vector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
; VPLANS-NEXT: <x1> vector loop: {
@@ -34,8 +34,8 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]>
; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]>
; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV_UPDATE]]>, vp<[[VF]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = active lane mask control flow vp<[[INC]]>, vp<[[TC]]>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-subvector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP_MASK:%.+]]> = wide active lane mask vp<[[INC]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = extract-vector-for-part vp<[[LANEMASK_LOOP_MASK]]>, ir<0>
; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]>
; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]>
; VPLANS-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
index 3226ed2ba779c..c40aad62202dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-alias-mask.ll
@@ -25,8 +25,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; FINAL-EMPTY:
; FINAL-NEXT: vector.ph:
-; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow ir<0>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; FINAL-NEXT: Successor(s): vector.body
; FINAL-EMPTY:
; FINAL-NEXT: vector.body:
@@ -41,8 +41,8 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; FINAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, vp<%index>
; FINAL-NEXT: WIDEN store ir<%ptr.c>, ir<%add>, vp<[[VP9]]>
; FINAL-NEXT: EMIT vp<%index.next> = add vp<%index>, vp<%num.active.lanes>
-; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<%index.next>, ir<%n>, ir<1>
-; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; FINAL-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<%n>, ir<1>
+; FINAL-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
; FINAL-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%extract.next.alm.part>
; FINAL-NEXT: EMIT branch-on-cond vp<[[VP10]]>
; FINAL-NEXT: Successor(s): middle.block, vector.body
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 9da97a2276b6a..c8d2572d21472 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -137,8 +137,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask control flow vp<%index.part.next>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-subvector-for-part vp<%active.lane.mask.entry>, ir<0>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -157,8 +157,8 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), vp<[[MASK]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>
; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<%index.next>, vp<[[VF]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask control flow vp<[[PART_IDX]]>, ir<%N>, ir<1>
-; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-subvector-for-part vp<%active.lane.mask.next>, ir<0>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<[[PART_IDX]]>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%extract.next.alm.part>
; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>
; CHECK-NEXT: No successors
>From 403bcaaba5f0019bbdf02fb375d2a7206dba9b01 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 22 Jul 2026 08:55:59 +0000
Subject: [PATCH 6/9] - Removed Multiplier from existing ActiveLaneMask
VPInstruction - Moved ICMP optimisation to optimizeForVFAndUF - Remove
preferTailFoldedLoop from selectInterleaveCount() - Add test with i32 trip
count
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +-
.../Transforms/Vectorize/VPlanLowering.cpp | 19 ++-
.../Transforms/Vectorize/VPlanPatternMatch.h | 6 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 32 ++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 53 ++++++--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
.../Transforms/Vectorize/VPlanVerifier.cpp | 5 -
.../AArch64/fixed-wide-lane-mask.ll | 115 ++++++++++++++++--
.../LoopVectorize/AArch64/masked-call.ll | 2 -
.../LoopVectorize/AArch64/pr73894.ll | 5 +-
.../AArch64/scalable-strict-fadd.ll | 99 ++++++++-------
.../AArch64/uniform-args-call-variants.ll | 78 ++++++------
.../AArch64/wide-alm-scalar-vf.ll | 71 +++++++++++
.../VPlan/buildvector-first-lane-only.ll | 2 +-
.../VPlan/early_exit_with_stores_vplan.ll | 6 +-
.../Vectorize/VPlanVerifierTest.cpp | 42 +++++++
17 files changed, 378 insertions(+), 168 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4e430e47ed659..c5ec1c1346f99 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3633,7 +3633,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Do not interleave tail-folded loops, as the overhead of multiple
// instructions to calculate the predicate is likely not beneficial.
// If an epilogue is not allowed for any other reason, do not interleave.
- if (!CM.isEpilogueAllowed() || CM.preferTailFoldedLoop())
+ if (!CM.isEpilogueAllowed())
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 9f8e4eb3c44b1..fc648a4350749 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1232,8 +1232,13 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
- // The size of the mask returned is VF * Multiplier (UF, third op).
+ // The size of the mask returned is VF.
+ // When unrolled, ActiveLaneMask is duplicated.
ActiveLaneMask,
+ // As above, but takes an additional operand (Multiplier). The size of
+ // the mask returned is VF * Multiplier (UF, op2).
+ // WideActiveLaneMask is used for control flow and is unrolled by widening,
+ // with one extract vector created per unroll part.
WideActiveLaneMask,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index e3b0ecc37c1dc..0a9227b0def6f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -150,17 +150,15 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
// TODO: Check if dropping the flags is needed.
TopRegion->clearCanonicalIVNUW(CanonicalIVIncrement);
DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
- // We can't use StartV directly in the WideActiveLaneMask
- // VPInstruction, since we have to take unrolling into account.
- // Each part needs to start at Part * VF
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
-
- // Create the WideActiveLaneMask instruction using the correct
- // start values.
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
+ // We can't use StartV directly in the WideActiveLaneMask
+ // VPInstruction, since we have to take unrolling into account.
+ // CanonicalIVIncrementForPart is needed so that ExtractVectorForPart
+ // is unrolled for each part, rather than each extract starting at 0.
auto *EntryIncrement =
Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
{StartV, VF}, {}, DL, "index.part.next");
@@ -224,12 +222,9 @@ void VPlanTransforms::materializeHeaderMask(
VPIRFlags::WrapFlagsTy(/*HasNUW=*/true, /*HasNSW=*/false)));
VPValue *Mask;
if (UseActiveLaneMask) {
- VPValue *ALMMultiplier =
- Plan.getConstantInt(LoopRegion->getCanonicalIVType(), 1);
- Mask = Builder.createNaryOp(
- VPInstruction::ActiveLaneMask,
- {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier}, nullptr,
- "active.lane.mask");
+ Mask = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+ {WideCanonicalIV, Plan.getTripCount()}, nullptr,
+ "active.lane.mask");
} else {
Mask = Builder.createICmp(CmpInst::ICMP_ULE, WideCanonicalIV,
Plan.getOrCreateBackedgeTakenCount());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 1fe1715dc35e2..c863d518c5464 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -454,12 +454,6 @@ m_ExtractPenultimateElement(const Op0_t &Op0) {
return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
}
-template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t>
-m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
-}
-
template <typename Op0_t, typename Op1_t, typename Op2_t>
inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 149db867d1c5e..19a12baa378dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -649,6 +649,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::FCmp:
case Instruction::ExtractElement:
case Instruction::Store:
+ case VPInstruction::ActiveLaneMask:
case VPInstruction::BranchOnCount:
case VPInstruction::BranchOnTwoConds:
case VPInstruction::FirstOrderRecurrenceSplice:
@@ -663,7 +664,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
return 2;
case Instruction::InsertElement:
case Instruction::Select:
- case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
return 3;
@@ -795,9 +795,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
// Get the original loop tripcount.
Value *ScalarTC = State.get(getOperand(1), VPLane(0));
+ unsigned Multiplier =
+ getOpcode() == VPInstruction::ActiveLaneMask
+ ? 1
+ : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
- unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
if (State.VF.isScalar() && Multiplier == 1)
return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC,
Name);
@@ -1116,23 +1120,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
if (Src->getType() == DstTy)
return Src;
- // If the VF is scalar & this is an extract of an active lane mask,
- // generate an ICMP directly.
- VPValue *Start, *TC;
- if (State.VF.isScalar() &&
- match(getOperand(0),
- m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
- m_VPValue()))) {
- Value *StartV = State.get(Start);
- if (Part > 0)
- StartV = Builder.CreateAdd(StartV, State.get(getOperand(1)));
- return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, StartV,
- State.get(TC));
- }
-
- auto *Idx = ConstantInt::get(Builder.getInt64Ty(),
- State.VF.getKnownMinValue() * Part);
- return Builder.CreateExtractVector(DstTy, Src, Idx);
+ return Builder.CreateExtractVector(
+ DstTy, Src, Builder.getInt64(State.VF.getKnownMinValue() * Part), Name);
}
default:
llvm_unreachable("Unsupported opcode for instruction");
@@ -1444,7 +1433,10 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
- unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ unsigned Multiplier =
+ getOpcode() == VPInstruction::ActiveLaneMask
+ ? 1
+ : cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8232957864c8b..e8b9de3d0b192 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2035,6 +2035,43 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
+static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
+ if (!BestVF.isScalar())
+ return false;
+
+ bool MadeChange = false;
+ VPBuilder Builder;
+ VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
+ VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
+
+ VPValue *Start, *TC;
+ uint64_t Idx;
+ for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (!match(&R, m_ExtractVectorForPart(
+ m_WideActiveLaneMask(m_VPValue(Start), m_VPValue(TC),
+ m_VPValue()),
+ m_ConstantInt(Idx))))
+ continue;
+
+ auto *Extract = cast<VPInstruction>(&R);
+ Builder.setInsertPoint(Extract);
+
+ if (Idx > 0)
+ Start = Builder.createAdd(
+ Start, Plan.getConstantInt(Start->getScalarType(), Idx));
+
+ VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC);
+ Extract->replaceAllUsesWith(ICmp);
+ Extract->eraseFromParent();
+ MadeChange = true;
+ }
+ }
+
+ return MadeChange;
+}
+
/// Try to simplify the branch condition of \p Plan. This may restrict the
/// resulting plan to \p BestVF and \p BestUF.
static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
@@ -2050,16 +2087,13 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
if (match(Term, m_BranchOnCount(
m_CombineOr(m_CanIVInc, m_c_Add(m_CanIVInc, m_LiveIn())),
m_VPValue())) ||
- match(Term, m_BranchOnCond(m_Not(m_ActiveLaneMask(
- m_VPValue(), m_VPValue(), m_VPValue())))) ||
match(Term,
m_BranchOnCond(m_Not(m_ExtractVectorForPart(
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()),
m_ZeroInt()))))) {
// Try to simplify the branch condition if VectorTC <= VF * UF when the
- // latch terminator is BranchOnCount, BranchOnCond(Not(ActiveLaneMask)) or
- // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask),
- // 0))
+ // latch terminator is BranchOnCount or
+ // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
const SCEV *VectorTripCount =
vputils::getSCEVExprForVPValue(&Plan.getVectorTripCount(), PSE);
if (isa<SCEVCouldNotCompute>(VectorTripCount))
@@ -2103,8 +2137,8 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange =
- simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
+ bool MadeChange = replaceMaskWithCompare(Plan, BestVF);
+ MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
if (MadeChange) {
@@ -3102,13 +3136,12 @@ static bool handleUncountableExitsWithSideEffects(
VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
VPValue *FirstActive = MaskBuilder.createFirstActiveLane(*Cond);
Type *IVScalarTy = IV->getScalarType();
- VPValue *ALMMultiplier = Plan.getConstantInt(IVScalarTy, 1);
VPValue *Zero = Plan.getZero(IVScalarTy);
FirstActive =
MaskBuilder.createScalarZExtOrTrunc(FirstActive, IVScalarTy, DebugLoc());
VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
- {Zero, FirstActive, ALMMultiplier},
- DebugLoc(), "uncountable.exit.mask");
+ {Zero, FirstActive}, DebugLoc(),
+ "uncountable.exit.mask");
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f3d3435962d8e..2b7a743abb6b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -481,7 +481,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
addUniformForAllParts(ALM);
- ALM->setOperand(2, Plan.getConstantInt(64, UF));
+ ALM->setOperand(2, getConstantInt(UF));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 68f4eb44d7ff5..ba4251f400df0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -200,11 +200,6 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
match(Mask, m_VPInstruction<VPInstruction::WideActiveLaneMask>()))
continue;
- if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())) ||
- match(Mask,
- m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
- continue;
-
CmpPredicate Pred;
VPValue *LHS, *RHS;
if (match(Mask, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) &&
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index bff2a61711124..090cb47dd315e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail \
@@ -84,14 +84,107 @@ for.end:
ret void
}
-attributes #0 = { nounwind "target-features"="+neon,+sve" }
+define void @fixed_wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 {
+; CHECK-UF1-LABEL: define void @fixed_wide_active_lane_mask_i32_tc(
+; CHECK-UF1-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF1-NEXT: entry:
+; CHECK-UF1-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF1-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF1-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF1: vector.memcheck:
+; CHECK-UF1-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF1-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF1-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; CHECK-UF1-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF1: vector.ph:
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[N]])
+; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF1: vector.body:
+; CHECK-UF1-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF1-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF1-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF1-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-UF1-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-UF1-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-UF1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF1: middle.block:
+;
+; CHECK-UF4-LABEL: define void @fixed_wide_active_lane_mask_i32_tc(
+; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF4: vector.memcheck:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 127
+; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 [[N]])
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 12)
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <4 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 4
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 8
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP6]], i64 12
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP6]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK3]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP8]], <4 x i1> [[ACTIVE_LANE_MASK4]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP9]], <4 x i1> [[ACTIVE_LANE_MASK5]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD6]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD7]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP13:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD8]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 4
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 8
+; CHECK-UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 12
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr align 8 [[TMP14]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr align 8 [[TMP15]], <4 x i1> [[ACTIVE_LANE_MASK3]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP12]], ptr align 8 [[TMP16]], <4 x i1> [[ACTIVE_LANE_MASK4]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP13]], ptr align 8 [[TMP17]], <4 x i1> [[ACTIVE_LANE_MASK5]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-UF4-NEXT: [[TMP18]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[TMP19]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP20]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-UF4-NEXT: [[TMP21]] = call <4 x i1> @llvm.vector.extract.v4i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 12)
+; CHECK-UF4-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0
+; CHECK-UF4-NEXT: [[TMP23:%.*]] = xor i1 [[TMP22]], true
+; CHECK-UF4-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ br label %for.body
-;.
-; CHECK-UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-UF1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-UF1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-;.
-; CHECK-UF4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK-UF4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK-UF4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-;.
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind "target-features"="+neon,+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 92bd724aeb415..890d0b0987240 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -996,7 +996,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1019,7 +1018,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index a01923d1ce361..ce49f94a4d43d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -12,13 +12,11 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[UMAX]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ true, [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT7:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[PRED_LOAD_CONTINUE5]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE5]] ]
@@ -44,7 +42,6 @@ define i32 @pr70988(ptr %src, i32 %n) {
; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]]
; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[UMAX]])
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index c79684470e794..f2a86a731a35b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -300,21 +300,22 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP18]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP19]], %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -331,13 +332,11 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP16]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP14]], <vscale x 8 x float> [[TMP15]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT10]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP17]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP18]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP19]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP22]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP23]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = xor i1 [[TMP20]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -1304,21 +1303,22 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -1347,13 +1347,11 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP16]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], <vscale x 8 x float> [[TMP23]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -1551,21 +1549,22 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-ORDERED-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul i64 [[TMP1]], 3
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY2:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP3]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP4]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP33:%.*]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; CHECK-ORDERED-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-ORDERED-TF: [[VECTOR_BODY]]:
; CHECK-ORDERED-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY2]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT15:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[TMP25]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK4:%.*]] = phi <vscale x 8 x i1> [ [[TMP26]], %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 8 x i1> [ [[TMP27]], %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[TMP33]], %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-ORDERED-TF-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP1]], 3
; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP1]]
; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP3]]
; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i64 [[TMP4]]
@@ -1594,13 +1593,11 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP16]], <vscale x 8 x float> splat (float -0.000000e+00)
; CHECK-ORDERED-TF-NEXT: [[TMP24]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP22]], <vscale x 8 x float> [[TMP23]])
; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = add i64 [[INDEX_NEXT]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT: [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP25]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP26]], i64 [[N]])
-; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP27]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; CHECK-ORDERED-TF-NEXT: [[TMP30]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 8)
+; CHECK-ORDERED-TF-NEXT: [[TMP31]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 16)
+; CHECK-ORDERED-TF-NEXT: [[TMP32]] = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv32i1(<vscale x 32 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 24)
; CHECK-ORDERED-TF-NEXT: [[TMP28:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-ORDERED-TF-NEXT: [[TMP29:%.*]] = xor i1 [[TMP28]], true
; CHECK-ORDERED-TF-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index 0a5a00af05e9d..91ff96cd7f2d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -43,30 +43,30 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @foo_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i64 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
-; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -127,30 +127,30 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP5]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD2]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = call <vscale x 2 x double> @bar_uniform(<vscale x 2 x double> [[WIDE_MASKED_LOAD3]], i32 [[UNIFORM]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; INTERLEAVE-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
-; INTERLEAVE-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; INTERLEAVE-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP11]], i64 0
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
-; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; INTERLEAVE: middle.block:
; INTERLEAVE-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
; INTERLEAVE: for.cond.cleanup:
@@ -198,7 +198,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: entry:
; INTERLEAVE-NEXT: br label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N]])
; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]]
; INTERLEAVE-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]]
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -226,7 +225,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE3]]
; INTERLEAVE: pred.store.continue3:
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; INTERLEAVE-NEXT: [[TMP11]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i64 [[INDEX_NEXT]], 1
; INTERLEAVE-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
new file mode 100644
index 0000000000000..f6174a0ee37bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-scalar-vf.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=1 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4
+
+target triple = "aarch64-unknown-linux"
+
+define void @wide_active_lane_mask_i32_tc(ptr %src, ptr %dst, i32 %n) #0 {
+; CHECK-UF4-LABEL: define void @wide_active_lane_mask_i32_tc(
+; CHECK-UF4-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-UF4-NEXT: entry:
+; CHECK-UF4-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
+; CHECK-UF4-NEXT: [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-UF4-NEXT: br label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-UF4: vector.memcheck:
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-UF4-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 15
+; CHECK-UF4-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-UF4: vector.ph:
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = icmp ult i32 0, [[N]]
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = icmp ult i32 1, [[N]]
+; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-UF4: vector.body:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE5:%.*]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_STORE_CONTINUE5]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi i1 [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_STORE_CONTINUE5]] ]
+; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK-UF4: pred.store.if:
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP4]], align 8
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = fmul double [[TMP5]], 3.000000e+00
+; CHECK-UF4-NEXT: store double [[TMP7]], ptr [[TMP6]], align 8
+; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE]]
+; CHECK-UF4: pred.store.continue:
+; CHECK-UF4-NEXT: br i1 [[ACTIVE_LANE_MASK3]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5]]
+; CHECK-UF4: pred.store.if4:
+; CHECK-UF4-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
+; CHECK-UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i32 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP9]], align 8
+; CHECK-UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds double, ptr [[DST]], i32 [[TMP8]]
+; CHECK-UF4-NEXT: [[TMP12:%.*]] = fmul double [[TMP10]], 3.000000e+00
+; CHECK-UF4-NEXT: store double [[TMP12]], ptr [[TMP11]], align 8
+; CHECK-UF4-NEXT: br label [[PRED_STORE_CONTINUE5]]
+; CHECK-UF4: pred.store.continue5:
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-UF4-NEXT: [[TMP13]] = icmp ult i32 [[INDEX_NEXT]], [[N]]
+; CHECK-UF4-NEXT: [[TMP14:%.*]] = add i32 [[INDEX_NEXT]], 1
+; CHECK-UF4-NEXT: [[TMP15]] = icmp ult i32 [[TMP14]], [[N]]
+; CHECK-UF4-NEXT: [[TMP16:%.*]] = xor i1 [[TMP13]], true
+; CHECK-UF4-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4: middle.block:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i32 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i32 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index ded3bc66016fe..b57af70695f6c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -17,7 +17,7 @@ define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 822bc1d4f3d93..64c8f2eaf9ecc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -28,7 +28,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
@@ -117,7 +117,7 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
@@ -202,7 +202,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>, ir<1>
; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index ebf7a0942ae2c..eda6157fde93f 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -514,4 +514,46 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
#endif
}
+TEST_F(VPIRVerifierTest, testVerifyLastActiveWideActiveLaneMask) {
+ const char *ModuleString =
+ "define i32 @f(ptr %A, i64 %N) {\n"
+ "entry:\n"
+ " br label %loop\n"
+ "loop:\n"
+ " %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n"
+ " %gep = getelementptr i32, ptr %A, i64 %iv\n"
+ " %ld = load i32, ptr %gep \n"
+ " %iv.next = add i64 %iv, 1\n"
+ " %exitcond = icmp eq i64 %iv.next, %N\n"
+ " br i1 %exitcond, label %exit, label %loop\n"
+ "exit:\n"
+ " ret i32 %ld\n"
+ "}\n";
+
+ Module &M = parseModule(ModuleString);
+
+ Function *F = M.getFunction("f");
+ BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
+ auto Plan = buildVPlan(LoopHeader);
+
+ VPValue *Start = Plan->getConstantInt(64, 0);
+ VPValue *TC = Plan->getTripCount();
+ VPValue *One = Plan->getConstantInt(64, 1);
+ Type *MaskTy = FixedVectorType::get(Type::getInt1Ty(*Ctx), 4);
+ Type *IndexTy = Plan->getDataLayout().getIndexType(Plan->getContext(), 0);
+
+ auto *WideALM =
+ new VPInstruction(VPInstruction::WideActiveLaneMask, {Start, TC, One}, {},
+ {}, DebugLoc(), "", MaskTy);
+ auto *Preheader = Plan->getVectorPreheader();
+ WideALM->insertBefore(*Preheader, Preheader->end());
+
+ auto *LastActiveLane =
+ new VPInstruction(VPInstruction::LastActiveLane, {WideALM}, {}, {},
+ DebugLoc(), "", IndexTy);
+ LastActiveLane->insertBefore(Plan->getMiddleBlock()->getTerminator());
+
+ EXPECT_TRUE(verifyVPlanIsValid(*Plan));
+}
+
} // namespace
>From 06e635f83bc47abed20aa390fffc04e32bf36300 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 27 Jul 2026 10:46:10 +0000
Subject: [PATCH 7/9] - Add a test with const trip count
---
.../LoopVectorize/AArch64/wide-alm-const.ll | 61 +++++++++++++++++++
1 file changed, 61 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
new file mode 100644
index 0000000000000..7243a4b56358d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=4 -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-UF4
+
+target triple = "aarch64-unknown-linux"
+
+define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonly %src) #0 {
+; CHECK-UF4-LABEL: define void @scalable_wide_active_lane_mask_double(
+; CHECK-UF4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-UF4-NEXT: [[ENTRY:.*:]]
+; CHECK-UF4-NEXT: [[CMP6:%.*]] = icmp sgt i64 9, 0
+; CHECK-UF4-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-UF4: [[FOR_BODY_PREHEADER]]:
+; CHECK-UF4-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-UF4: [[VECTOR_PH]]:
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 9)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-UF4: [[VECTOR_BODY]]:
+; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 4
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK2]], <4 x double> poison)
+; CHECK-UF4-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[WIDE_MASKED_LOAD3]], splat (double 3.000000e+00)
+; CHECK-UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-UF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP4]], i64 4
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP2]], ptr align 8 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-UF4-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP3]], ptr align 8 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 9)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-UF4-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-UF4-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-UF4-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; CHECK-UF4-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UF4: [[MIDDLE_BLOCK]]:
+;
+entry:
+ %cmp6 = icmp sgt i64 9, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i64 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 9
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind vscale_range(1,16) "target-features"="+sve2p1" }
>From 63b1e31f7475a074c48397896a34e89ebd4d2efa Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 5 Aug 2026 12:38:45 +0000
Subject: [PATCH 8/9] - Swap checks for ActiveLaneMask with WideActiveLaneMask
in VPRecipes - Replace unsigned with uint64_t for getZExtValue() - Add
comment to describe replaceMaskWithCompare
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +++
3 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fc648a4350749..3f2158a1e165c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1232,7 +1232,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Creates a mask where each lane is active (true) whilst the current
// counter (first operand + index) is less than the second operand. i.e.
// mask[i] = icmpt ult (op0 + i), op1
- // The size of the mask returned is VF.
+ // ActiveLaneMask is used for tail-folding, with the exception of the
+ // DataAndControlFlow style. The size of the mask returned is VF.
// When unrolled, ActiveLaneMask is duplicated.
ActiveLaneMask,
// As above, but takes an additional operand (Multiplier). The size of
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 19a12baa378dd..b6cfe8724b88d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -795,10 +795,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
// Get the original loop tripcount.
Value *ScalarTC = State.get(getOperand(1), VPLane(0));
- unsigned Multiplier =
- getOpcode() == VPInstruction::ActiveLaneMask
- ? 1
- : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ uint64_t Multiplier =
+ getOpcode() == VPInstruction::WideActiveLaneMask
+ ? cast<VPConstantInt>(getOperand(2))->getZExtValue()
+ : 1;
// If this part of the active lane mask is scalar, generate the CMP directly
// to avoid unnecessary extracts.
@@ -1115,7 +1115,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::ExtractVectorForPart: {
Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
- unsigned Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
+ uint64_t Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
if (Src->getType() == DstTy)
return Src;
@@ -1433,10 +1433,10 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
Type *ArgTy = getOperand(0)->getScalarType();
- unsigned Multiplier =
- getOpcode() == VPInstruction::ActiveLaneMask
- ? 1
- : cast<VPConstantInt>(getOperand(2))->getZExtValue();
+ uint64_t Multiplier =
+ getOpcode() == VPInstruction::WideActiveLaneMask
+ ? cast<VPConstantInt>(getOperand(2))->getZExtValue()
+ : 1;
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e8b9de3d0b192..93a38fa995747 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2035,6 +2035,9 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
return SE.isKnownPredicate(CmpInst::ICMP_EQ, VectorTripCount, C);
}
+// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
+// and the source is a WideActiveLaneMask. The unused mask is removed later
+// when removing dead recipes.
static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
if (!BestVF.isScalar())
return false;
>From d0016b6663305084ad6338e0d7293fd0e5f264c7 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 7 Aug 2026 15:18:57 +0000
Subject: [PATCH 9/9] - Address new feedback: - Remove
CanonicalIVIncrementForPart in addVPLaneMaskPhiAndUpdateExitBranch and
ensure isUniformAcrossVFsAndUFs returns false for ExtractVectorForPart -
Use overflowing add in replaceMaskWithCompare
---
.../Transforms/Vectorize/VPlanLowering.cpp | 10 +-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +++--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 1 -
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 14 +++++------
.../AArch64/sve-wide-lane-mask.ll | 24 +++++++++----------
.../VPlan/AArch64/sve-tail-folding-forced.ll | 3 +--
.../VPlan/AArch64/vplan-printing.ll | 3 +--
7 files changed, 26 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 0a9227b0def6f..d009c393bc23e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -155,19 +155,11 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
VPValue *TC = Plan.getTripCount();
VPValue *VF = &Plan.getVF();
- // We can't use StartV directly in the WideActiveLaneMask
- // VPInstruction, since we have to take unrolling into account.
- // CanonicalIVIncrementForPart is needed so that ExtractVectorForPart
- // is unrolled for each part, rather than each extract starting at 0.
- auto *EntryIncrement =
- Builder.createOverflowingOp(VPInstruction::CanonicalIVIncrementForPart,
- {StartV, VF}, {}, DL, "index.part.next");
-
// Create the wide active lane mask instruction in the VPlan preheader.
VPValue *ALMMultiplier =
Plan.getConstantInt(TopRegion->getCanonicalIVType(), 1);
auto *EntryALM = Builder.createNaryOp(VPInstruction::WideActiveLaneMask,
- {EntryIncrement, TC, ALMMultiplier}, DL,
+ {StartV, TC, ALMMultiplier}, DL,
"active.lane.mask.entry");
EntryALM = Builder.createNaryOp(VPInstruction::ExtractVectorForPart,
{EntryALM, Plan.getConstantInt(64, 0)}, DL,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 93a38fa995747..8e0f929c245d1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2062,8 +2062,10 @@ static bool replaceMaskWithCompare(VPlan &Plan, ElementCount BestVF) {
Builder.setInsertPoint(Extract);
if (Idx > 0)
- Start = Builder.createAdd(
- Start, Plan.getConstantInt(Start->getScalarType(), Idx));
+ Start = Builder.createOverflowingOp(
+ Instruction::Add,
+ {Start, Plan.getConstantInt(Start->getScalarType(), Idx)},
+ {false, false});
VPValue *ICmp = Builder.createICmp(CmpInst::ICMP_ULT, Start, TC);
Extract->replaceAllUsesWith(ICmp);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 2b7a743abb6b6..21ea2a6b74ff5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -480,7 +480,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
if (match(&R,
m_WideActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue()))) {
auto *ALM = cast<VPInstruction>(&R);
- addUniformForAllParts(ALM);
ALM->setOperand(2, getConstantInt(UF));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c01ddfbd24263..e356b2cf9d955 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -465,13 +465,13 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
const VPRecipeBase *R = V->getDefiningRecipe();
const VPBasicBlock *VPBB = R ? R->getParent() : nullptr;
const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr;
- if (VPBB) {
- if ((VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
- if (match(V->getDefiningRecipe(),
- m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()))
- return false;
- return all_of(R->operands(), isUniformAcrossVFsAndUFs);
- }
+ if (VPBB &&
+ (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
+ if (match(R,
+ m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()) ||
+ match(R, m_ExtractVectorForPart(m_VPValue(), m_VPValue())))
+ return false;
+ return all_of(R->operands(), isUniformAcrossVFsAndUFs);
}
return TypeSwitch<const VPRecipeBase *, bool>(R)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index f77faf7eac54e..8c213d3aa528f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -40,16 +40,16 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 0, i64 [[N]])
; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
-; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 32)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 48)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 16 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
@@ -142,16 +142,16 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl
; CHECK-UF4-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
; CHECK-UF4-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
-; CHECK-UF4-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
-; CHECK-UF4-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 6)
; CHECK-UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF4: vector.body:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <vscale x 2 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP1]], 1
; CHECK-UF4-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP1]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
index 86ea9eb5a2e9e..8fd52c63c5994 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve-tail-folding-forced.ll
@@ -18,8 +18,7 @@ target triple = "aarch64-unknown-linux-gnu"
; VPLANS-NEXT: Successor(s): scalar.ph, vector.ph
; VPLANS-EMPTY:
; VPLANS-NEXT: vector.ph:
-; VPLANS-NEXT: EMIT vp<[[VF_PER_PART:%.+]]> = VF * Part + ir<0>
-; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask vp<[[VF_PER_PART]]>, vp<[[TC]]>
+; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%.+]]> = wide active lane mask ir<0>, vp<[[TC]]>, ir<1>
; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY_EXTRACT:%.+]]> = extract-vector-for-part vp<[[LANEMASK_ENTRY]]>, ir<0>
; VPLANS-NEXT: Successor(s): vector loop
; VPLANS-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index c8d2572d21472..39d3a67fa3d08 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -136,8 +136,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
-; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>
-; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask vp<%index.part.next>, ir<%N>, ir<1>
+; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<%N>, ir<1>
; CHECK-NEXT: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
More information about the llvm-commits
mailing list