[llvm] [LV] Enable wide active lane masks when tail-folding if preferred (PR #193757)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 08:11:35 PDT 2026
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/193757
>From 9734d21f2de43627a2f0205c8fc461e0428fc953 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 14 Apr 2026 15:31:07 +0000
Subject: [PATCH 1/4] [LV] Enable wide active lane masks when tail-folding if
preferred
Adds a new TTI hook which returns true if the target prefers wide
active lane masks. For AArch64 this returns true for targets with
SVE2p1 or SME2, as this means the whilelo (predicate pair)
instruction is available.
When preferWideActiveLaneMasks returns true, wide active lane masks
will be considered when tail-folding is also enabled.
This does not affect the default behaviour of any target.
---
.../llvm/Analysis/TargetTransformInfo.h | 12 ++
.../llvm/Analysis/TargetTransformInfoImpl.h | 2 +
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +
.../AArch64/AArch64TargetTransformInfo.h | 5 +
.../Transforms/Vectorize/LoopVectorize.cpp | 28 +++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 +-
.../Transforms/Vectorize/VPlanTransforms.h | 5 +-
.../AArch64/fixed-wide-lane-mask.ll | 4 +-
.../AArch64/sve-wide-lane-mask.ll | 8 +-
.../AArch64/wide-lane-mask-flags.ll | 121 ++++++++++++++++++
.../LoopVectorize/ARM/active-lane-mask.ll | 2 +-
11 files changed, 184 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 333c5e4868395..683eaa4133b80 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -253,6 +253,16 @@ enum class TailFoldingStyle {
DataWithEVL,
};
+enum class WideActiveLaneMask {
+ // Do not consider using wide active lane masks.
+ Disable,
+ // Considered when the TailFoldingStyle is DataAndControlFlow and
+ // preferWideActiveLaneMasks() returns true for the target.
+ Default,
+ // Always consider using wide active lane masks.
+ Force,
+};
+
struct TailFoldingInfo {
TargetLibraryInfo *TLI;
LoopVectorizationLegality *LVL;
@@ -1961,6 +1971,8 @@ class TargetTransformInfo {
/// processing \p Iters scalar iterations per vector iteration.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const;
+ LLVM_ABI bool preferWideActiveLaneMasks() const;
+
/// \returns True if the loop vectorizer should discard any VFs where the
/// maximum register pressure exceeds getNumberOfRegisters.
LLVM_ABI bool shouldConsiderVectorizationRegPressure() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 49349a22e21e9..dfd14de1003df 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1163,6 +1163,8 @@ class TargetTransformInfoImplBase {
return getMaxInterleaveFactor(Iters) > 1;
}
+ virtual bool preferWideActiveLaneMasks() const { return false; }
+
virtual bool shouldConsiderVectorizationRegPressure() const { return false; }
virtual bool shouldExpandReduction(const IntrinsicInst *II) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index e1ab90a8e046c..74207de53139f 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1481,6 +1481,10 @@ bool TargetTransformInfo::preferEpilogueVectorization(
return TTIImpl->preferEpilogueVectorization(Iters);
}
+bool TargetTransformInfo::preferWideActiveLaneMasks() const {
+ return TTIImpl->preferWideActiveLaneMasks();
+}
+
bool TargetTransformInfo::shouldConsiderVectorizationRegPressure() const {
return TTIImpl->shouldConsiderVectorizationRegPressure();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 8125b7ffcae43..9997686b33451 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -477,6 +477,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override;
+ bool preferWideActiveLaneMasks() const override {
+ return ST->isSVEorStreamingSVEAvailable() &&
+ (ST->hasSVE2p1() || ST->hasSME2());
+ }
+
bool supportsScalableVectors() const override {
return ST->isSVEorStreamingSVEAvailable();
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e912751525fc7..45f3bfe8e3f6f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -236,10 +236,18 @@ static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"Use predicated EVL instructions for tail folding. If EVL "
"is unsupported, fallback to data-without-lane-mask.")));
-cl::opt<bool> llvm::EnableWideActiveLaneMask(
- "enable-wide-lane-mask", cl::init(false), cl::Hidden,
+cl::opt<WideActiveLaneMask> llvm::EnableWideActiveLaneMask(
+ "enable-wide-lane-mask",
cl::desc("Enable use of wide lane masks when used for control flow in "
- "tail-folded loops"));
+ "tail-folded loops"),
+ cl::init(WideActiveLaneMask::Default),
+ cl::values(clEnumValN(WideActiveLaneMask::Default, "default",
+ "Decision to use wide active lane masks based on "
+ "target preference."),
+ clEnumValN(WideActiveLaneMask::Disable, "disable",
+ "Use of wide active lane masks disabled."),
+ clEnumValN(WideActiveLaneMask::Force, "force",
+ "Always use wide active lane masks where possible")));
static cl::opt<bool> EnableInterleavedMemAccesses(
"enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
@@ -1229,10 +1237,16 @@ class LoopVectorizationCostModel {
/// Returns true if the use of wide lane masks is requested and the loop is
/// using tail-folding with a lane mask for control flow.
bool useWideActiveLaneMask() const {
- if (!EnableWideActiveLaneMask)
+ switch (EnableWideActiveLaneMask.getValue()) {
+ case (WideActiveLaneMask::Disable):
return false;
-
- return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
+ case (WideActiveLaneMask::Force):
+ return true;
+ case (WideActiveLaneMask::Default):
+ return TTI.preferWideActiveLaneMasks() &&
+ getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
+ }
+ llvm_unreachable("invalid enum");
}
/// Returns true if the instructions in this block requires predication
@@ -6190,7 +6204,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
BestVF, BestUF, PSE);
RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
- PSE);
+ PSE, TTI);
RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
if (EpilogueVecKind == EpilogueVectorizationKind::None)
RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 11e5b18bca871..03b4c1a2ce458 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2135,8 +2135,12 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
/// new extracts from the first active lane mask, which has it's last
/// operand (multiplier) set to UF.
static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
- unsigned UF) {
- if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1)
+ unsigned UF,
+ const TargetTransformInfo &TTI) {
+ if (EnableWideActiveLaneMask == WideActiveLaneMask::Disable ||
+ (EnableWideActiveLaneMask == WideActiveLaneMask::Default &&
+ !TTI.preferWideActiveLaneMasks()) ||
+ !VF.isVector() || UF == 1)
return false;
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -2321,11 +2325,12 @@ bool VPlanTransforms::simplifyKnownEVL(VPlan &Plan, ElementCount VF,
void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
unsigned BestUF,
- PredicatedScalarEvolution &PSE) {
+ PredicatedScalarEvolution &PSE,
+ const TargetTransformInfo &TTI) {
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF);
+ bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF, TTI);
MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0cf095cee313f..2cd9781bd18a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -39,7 +39,7 @@ class VPRecipeBuilder;
struct VFRange;
LLVM_ABI_FOR_TEST extern cl::opt<bool> VerifyEachVPlan;
-LLVM_ABI_FOR_TEST extern cl::opt<bool> EnableWideActiveLaneMask;
+LLVM_ABI_FOR_TEST extern cl::opt<WideActiveLaneMask> EnableWideActiveLaneMask;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_ABI_FOR_TEST extern cl::opt<bool> VPlanPrintAfterAll;
@@ -246,7 +246,8 @@ struct VPlanTransforms {
/// resulting plan to \p BestVF and \p BestUF.
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
unsigned BestUF,
- PredicatedScalarEvolution &PSE);
+ PredicatedScalarEvolution &PSE,
+ const TargetTransformInfo &TTI);
/// Try to simplify VPInstruction::ExplicitVectorLength recipes when the AVL
/// is known to be <= VF, replacing them with the AVL directly.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index 17cc76c52c7e3..1228a8e76445f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force \
; RUN: -force-vector-width=4 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
target triple = "aarch64-unknown-linux"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 2e6e3a41addc5..7ff5084fad9b8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
-; RUN: opt -S --passes=loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix CHECK-TF
-; RUN: opt -S --passes=forceattrs,loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force < %s | FileCheck %s -check-prefix CHECK-TF
+; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
target triple = "aarch64-unknown-linux"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
new file mode 100644
index 0000000000000..be224fe1c9a7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 6
+
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=default -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=force -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=default -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-NO-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=disable -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-NO-WIDEALM
+
+target triple = "aarch64-unknown-linux"
+
+define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-WIDEALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-WIDEALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-WIDEALM-NEXT: [[ENTRY:.*:]]
+; CHECK-WIDEALM-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-WIDEALM-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-WIDEALM: [[FOR_BODY_PREHEADER]]:
+; CHECK-WIDEALM-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-WIDEALM: [[VECTOR_PH]]:
+; CHECK-WIDEALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-WIDEALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-WIDEALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-WIDEALM-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-WIDEALM-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-WIDEALM-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-WIDEALM: [[VECTOR_BODY]]:
+; CHECK-WIDEALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-WIDEALM-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 [[TMP1]]
+; CHECK-WIDEALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-WIDEALM-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; CHECK-WIDEALM-NEXT: [[TMP7:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-WIDEALM-NEXT: [[TMP8:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD2]], splat (double 3.000000e+00)
+; CHECK-WIDEALM-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-WIDEALM-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; CHECK-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-WIDEALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-WIDEALM-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-WIDEALM-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-WIDEALM-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; CHECK-WIDEALM-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-WIDEALM-NEXT: br i1 [[TMP14]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-WIDEALM: [[FOR_END_LOOPEXIT]]:
+;
+; CHECK-NO-WIDEALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-NO-WIDEALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NO-WIDEALM-NEXT: [[ENTRY:.*:]]
+; CHECK-NO-WIDEALM-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NO-WIDEALM-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-NO-WIDEALM: [[FOR_BODY_PREHEADER]]:
+; CHECK-NO-WIDEALM-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NO-WIDEALM: [[VECTOR_PH]]:
+; CHECK-NO-WIDEALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NO-WIDEALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NO-WIDEALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NO-WIDEALM: [[VECTOR_BODY]]:
+; CHECK-NO-WIDEALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NO-WIDEALM-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 [[TMP1]]
+; CHECK-NO-WIDEALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-NO-WIDEALM-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; CHECK-NO-WIDEALM-NEXT: [[TMP5:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-NO-WIDEALM-NEXT: [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD3]], splat (double 3.000000e+00)
+; CHECK-NO-WIDEALM-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NO-WIDEALM-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-NO-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NO-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP6]], ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-NO-WIDEALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-NO-WIDEALM-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NO-WIDEALM-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-NO-WIDEALM-NEXT: br i1 [[TMP11]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NO-WIDEALM: [[FOR_END_LOOPEXIT]]:
+;
+entry:
+ %cmp6 = icmp sgt i64 %n, 0
+ br i1 %cmp6, label %for.body, label %for.end
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds double, ptr %src, i64 %iv
+ %ld = load double, ptr %arrayidx1
+ %mul = fmul double %ld, 3.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { nounwind vscale_range(1,16) }
+;.
+; CHECK-WIDEALM: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-WIDEALM: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-WIDEALM: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
+; CHECK-NO-WIDEALM: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NO-WIDEALM: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NO-WIDEALM: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
index 6f82b9301fb45..37bb827f893a1 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -S | FileCheck %s
target triple = "thumbv8.1m.main-arm-unknown-eabihf"
>From d6e91fd4335f8af477d181336631337fa86bfa47 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 23 Apr 2026 14:13:18 +0000
Subject: [PATCH 2/4] - Add comment above preferWideActiveLaneMasks in
TargetTransformInfo.h
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 683eaa4133b80..caef986139258 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1971,6 +1971,9 @@ class TargetTransformInfo {
/// processing \p Iters scalar iterations per vector iteration.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const;
+ /// \returns True if the loop vectorizer should prefer using a single wide
+ /// active lane mask of size VF * UF in tail-folded loops, rather than one
+ /// active lane mask per part.
LLVM_ABI bool preferWideActiveLaneMasks() const;
/// \returns True if the loop vectorizer should discard any VFs where the
>From 7c8372e1d51b0a5735bcb6450742a7554f6b76ac Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 23 Apr 2026 15:10:22 +0000
Subject: [PATCH 3/4] - Run update_test_checks.py on wide-lane-mask-flags.ll
after rebase
---
.../Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
index be224fe1c9a7d..6418bc21733a0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
@@ -48,7 +48,7 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-WIDEALM-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
; CHECK-WIDEALM-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
-; CHECK-WIDEALM-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; CHECK-WIDEALM-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i64 0
; CHECK-WIDEALM-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
; CHECK-WIDEALM-NEXT: br i1 [[TMP14]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-WIDEALM: [[FOR_END_LOOPEXIT]]:
@@ -85,7 +85,7 @@ define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src,
; CHECK-NO-WIDEALM-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP9]], i64 [[N]])
-; CHECK-NO-WIDEALM-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NO-WIDEALM-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NO-WIDEALM-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
; CHECK-NO-WIDEALM-NEXT: br i1 [[TMP11]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-NO-WIDEALM: [[FOR_END_LOOPEXIT]]:
>From 1ebc8092e16460d5a6944119b2b28c800b7a7ac8 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 24 Apr 2026 17:09:02 +0000
Subject: [PATCH 4/4] - Added getActiveLaneMaskCost function - Include the cost
of extracts when NumResults > 1 - Compare cost of wide vs narrow lane mask
after selectInterleaveCount
---
.../llvm/Analysis/TargetTransformInfo.h | 13 +-
.../llvm/Analysis/TargetTransformInfoImpl.h | 9 +-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 61 ++++++---
llvm/lib/Analysis/TargetTransformInfo.cpp | 17 ++-
.../AArch64/AArch64TargetTransformInfo.cpp | 126 +++++++++++-------
.../AArch64/AArch64TargetTransformInfo.h | 10 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 28 ++--
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 5 +
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 ++++---
.../Target/RISCV/RISCVTargetTransformInfo.h | 5 +
.../Transforms/Vectorize/LoopVectorize.cpp | 38 +++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 22 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 1 +
.../CostModel/AArch64/sve-intrinsics.ll | 49 ++-----
.../AArch64/wide-lane-mask-flags.ll | 2 +
.../LoopVectorize/ARM/active-lane-mask.ll | 69 ++++++++--
.../LoopVectorize/RISCV/active-lane-mask.ll | 97 ++++++++++++++
18 files changed, 431 insertions(+), 171 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/active-lane-mask.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index caef986139258..e8dd046f6837e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -256,8 +256,7 @@ enum class TailFoldingStyle {
enum class WideActiveLaneMask {
// Do not consider using wide active lane masks.
Disable,
- // Considered when the TailFoldingStyle is DataAndControlFlow and
- // preferWideActiveLaneMasks() returns true for the target.
+ // Considered when the TailFoldingStyle is DataAndControlFlow.
Default,
// Always consider using wide active lane masks.
Force,
@@ -1761,6 +1760,11 @@ class TargetTransformInfo {
Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF = FastMathFlags(),
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
+ LLVM_ABI InstructionCost getActiveLaneMaskCost(
+ Type *ResTy, Type *ArgTy, FastMathFlags FMF = FastMathFlags(),
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+ unsigned NumResults = 1) const;
+
/// Calculate the cost of an extended reduction pattern, similar to
/// getArithmeticReductionCost of an Add/Sub reduction with multiply and
/// optional extensions. This is the cost of as:
@@ -1971,11 +1975,6 @@ class TargetTransformInfo {
/// processing \p Iters scalar iterations per vector iteration.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const;
- /// \returns True if the loop vectorizer should prefer using a single wide
- /// active lane mask of size VF * UF in tail-folded loops, rather than one
- /// active lane mask per part.
- LLVM_ABI bool preferWideActiveLaneMasks() const;
-
/// \returns True if the loop vectorizer should discard any VFs where the
/// maximum register pressure exceeds getNumberOfRegisters.
LLVM_ABI bool shouldConsiderVectorizationRegPressure() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index dfd14de1003df..414789768a045 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1010,6 +1010,13 @@ class TargetTransformInfoImplBase {
return 1;
}
+ virtual InstructionCost getActiveLaneMaskCost(Type *ResTy, Type *ArgTy,
+ FastMathFlags,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const {
+ return NumResults;
+ }
+
virtual InstructionCost
getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy,
VectorType *Ty, std::optional<FastMathFlags> FMF,
@@ -1163,8 +1170,6 @@ class TargetTransformInfoImplBase {
return getMaxInterleaveFactor(Iters) > 1;
}
- virtual bool preferWideActiveLaneMasks() const { return false; }
-
virtual bool shouldConsiderVectorizationRegPressure() const { return false; }
virtual bool shouldExpandReduction(const IntrinsicInst *II) const {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e70d81953cd5d..42f0bb3e64734 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2164,6 +2164,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
case Intrinsic::get_active_lane_mask:
+ return thisT()->getActiveLaneMaskCost(RetTy, ICA.getArgTypes()[0],
+ ICA.getFlags(), CostKind,
+ /* NumResults */ 1);
case Intrinsic::experimental_vector_match:
case Intrinsic::experimental_vector_histogram_add:
case Intrinsic::experimental_vector_histogram_uadd_sat:
@@ -2605,27 +2608,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
Cost *= PtrsTy->getNumElements();
return Cost;
}
- case Intrinsic::get_active_lane_mask: {
- Type *ArgTy = ICA.getArgTypes()[0];
- EVT ResVT = getTLI()->getValueType(DL, RetTy, true);
- EVT ArgVT = getTLI()->getValueType(DL, ArgTy, true);
-
- // If we're not expanding the intrinsic then we assume this is cheap
- // to implement.
- if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgVT))
- return getTypeLegalizationCost(RetTy).first;
-
- // Create the expanded types that will be used to calculate the uadd_sat
- // operation.
- Type *ExpRetTy =
- VectorType::get(ArgTy, cast<VectorType>(RetTy)->getElementCount());
- IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF);
- InstructionCost Cost =
- thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
- Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, RetTy,
- CmpInst::ICMP_ULT, CostKind);
- return Cost;
- }
+ case Intrinsic::get_active_lane_mask:
+ return thisT()->getActiveLaneMaskCost(RetTy, ICA.getArgTypes()[0],
+ ICA.getFlags(), CostKind,
+ /* NumResults */ 1);
case Intrinsic::experimental_memset_pattern:
// This cost is set to match the cost of the memset_pattern16 libcall.
// It should likely be re-evaluated after migration to this intrinsic
@@ -3426,6 +3412,39 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
CostKind, 0, nullptr, nullptr);
}
+ InstructionCost getActiveLaneMaskCost(Type *ResTy, Type *ArgTy,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const override {
+ EVT ResVT = getTLI()->getValueType(DL, ResTy, true);
+ EVT ArgVT = getTLI()->getValueType(DL, ArgTy, true);
+
+ InstructionCost ExtractCost = 0;
+ if (NumResults > 1) {
+ auto ResSubTy = VectorType::getOneNthElementsVectorType(
+ cast<VectorType>(ResTy), NumResults);
+ IntrinsicCostAttributes ICA(Intrinsic::vector_extract, ResSubTy,
+ {ResTy, ArgTy});
+ ExtractCost = getIntrinsicInstrCost(ICA, CostKind) * NumResults;
+ }
+
+ // If we're not expanding the intrinsic then we assume this is cheap
+ // to implement.
+ if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgVT))
+ return getTypeLegalizationCost(ResTy).first + ExtractCost;
+
+ // Create the expanded types that will be used to calculate the uadd_sat
+ // operation.
+ Type *ExpRetTy =
+ VectorType::get(ArgTy, cast<VectorType>(ResTy)->getElementCount());
+ IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF);
+ InstructionCost Cost =
+ thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
+ Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, ResTy,
+ CmpInst::ICMP_ULT, CostKind);
+ return Cost + ExtractCost;
+ }
+
InstructionCost
getExtendedReductionCost(unsigned Opcode, bool IsUnsigned, Type *ResTy,
VectorType *Ty, std::optional<FastMathFlags> FMF,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 74207de53139f..175266a46bf2e 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1326,6 +1326,19 @@ InstructionCost TargetTransformInfo::getMinMaxReductionCost(
return Cost;
}
+InstructionCost TargetTransformInfo::getActiveLaneMaskCost(
+ Type *ResTy, Type *ArgTy, FastMathFlags FMF, TTI::TargetCostKind CostKind,
+ unsigned NumResults) const {
+ // Return Invalid if ResTy cannot be split into the number of parts requested.
+ if (!cast<VectorType>(ResTy)->getElementCount().isKnownMultipleOf(NumResults))
+ return InstructionCost::getInvalid();
+
+ InstructionCost Cost =
+ TTIImpl->getActiveLaneMaskCost(ResTy, ArgTy, FMF, CostKind, NumResults);
+ assert(Cost >= 0 && "TTI should not produce negative costs!");
+ return Cost;
+}
+
InstructionCost TargetTransformInfo::getExtendedReductionCost(
unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
@@ -1481,10 +1494,6 @@ bool TargetTransformInfo::preferEpilogueVectorization(
return TTIImpl->preferEpilogueVectorization(Iters);
}
-bool TargetTransformInfo::preferWideActiveLaneMasks() const {
- return TTIImpl->preferWideActiveLaneMasks();
-}
-
bool TargetTransformInfo::shouldConsiderVectorizationRegPressure() const {
return TTIImpl->shouldConsiderVectorizationRegPressure();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index aff89e00523c0..8a76148237e07 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1013,54 +1013,6 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
break;
return TyL.first + ExtraCost;
}
- case Intrinsic::get_active_lane_mask: {
- auto RetTy = cast<VectorType>(ICA.getReturnType());
- EVT RetVT = getTLI()->getValueType(DL, RetTy);
- EVT OpVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
- if (getTLI()->shouldExpandGetActiveLaneMask(RetVT, OpVT))
- break;
-
- if (RetTy->isScalableTy()) {
- if (TLI->getTypeAction(RetTy->getContext(), RetVT) !=
- TargetLowering::TypeSplitVector)
- break;
-
- auto LT = getTypeLegalizationCost(RetTy);
- InstructionCost Cost = LT.first;
- // When SVE2p1 or SME2 is available, we can halve getTypeLegalizationCost
- // as get_active_lane_mask may lower to the sve_whilelo_x2 intrinsic, e.g.
- // nxv32i1 = get_active_lane_mask(base, idx) ->
- // {nxv16i1, nxv16i1} = sve_whilelo_x2(base, idx)
- if (ST->hasSVE2p1() || ST->hasSME2()) {
- Cost /= 2;
- if (Cost == 1)
- return Cost;
- }
-
- // If more than one whilelo intrinsic is required, include the extra cost
- // required by the saturating add & select required to increment the
- // start value after the first intrinsic call.
- Type *OpTy = ICA.getArgTypes()[0];
- IntrinsicCostAttributes AddAttrs(Intrinsic::uadd_sat, OpTy, {OpTy, OpTy});
- InstructionCost SplitCost = getIntrinsicInstrCost(AddAttrs, CostKind);
- Type *CondTy = OpTy->getWithNewBitWidth(1);
- SplitCost += getCmpSelInstrCost(Instruction::Select, OpTy, CondTy,
- CmpInst::ICMP_UGT, CostKind);
- return Cost + (SplitCost * (Cost - 1));
- } else if (!getTLI()->isTypeLegal(RetVT)) {
- // We don't have enough context at this point to determine if the mask
- // is going to be kept live after the block, which will force the vXi1
- // type to be expanded to legal vectors of integers, e.g. v4i1->v4i32.
- // For now, we just assume the vectorizer created this intrinsic and
- // the result will be the input for a PHI. In this case the cost will
- // be extremely high for fixed-width vectors.
- // NOTE: getScalarizationOverhead returns a cost that's far too
- // pessimistic for the actual generated codegen. In reality there are
- // two instructions generated per lane.
- return cast<FixedVectorType>(RetTy)->getNumElements() * 2;
- }
- break;
- }
case Intrinsic::experimental_vector_match: {
auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
EVT SearchVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
@@ -5704,6 +5656,84 @@ AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
return LegalizationCost + /*Cost of horizontal reduction*/ 2;
}
+InstructionCost AArch64TTIImpl::getActiveLaneMaskCost(
+ Type *ResTy, Type *ArgTy, FastMathFlags FMF, TTI::TargetCostKind CostKind,
+ unsigned NumResults) const {
+ if (cast<VectorType>(ResTy)->getElementCount() ==
+ ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
+ EVT RetVT = getTLI()->getValueType(DL, ResTy);
+ EVT OpVT = getTLI()->getValueType(DL, ArgTy);
+ if (ST->hasSVE2p1() || ST->hasSME2())
+ NumResults = (NumResults + 1) / 2;
+
+ if (getTLI()->shouldExpandGetActiveLaneMask(RetVT, OpVT))
+ return BaseT::getActiveLaneMaskCost(ResTy, ArgTy, FMF, CostKind,
+ NumResults);
+
+ InstructionCost ExtractCost = 0;
+ if (NumResults > 1) {
+ auto ResSubTy = VectorType::getOneNthElementsVectorType(
+ cast<VectorType>(ResTy), NumResults);
+ RetVT = getTLI()->getValueType(DL, ResSubTy);
+ if (ResSubTy->getElementCount() == ElementCount::getScalable(1))
+ return InstructionCost::getInvalid();
+
+ if (!ST->hasSVE2p1() && !ST->hasSME2()) {
+ IntrinsicCostAttributes ICA(Intrinsic::vector_extract, ResSubTy,
+ {ResTy, ArgTy});
+ ExtractCost = getIntrinsicInstrCost(ICA, CostKind) * NumResults;
+ }
+ }
+
+ if (ResTy->isScalableTy()) {
+ if (TLI->getTypeAction(ResTy->getContext(), RetVT) !=
+ TargetLowering::TypeSplitVector)
+ return BaseT::getActiveLaneMaskCost(ResTy, ArgTy, FMF, CostKind,
+ NumResults) +
+ ExtractCost;
+
+ auto LT = getTypeLegalizationCost(ResTy);
+ InstructionCost Cost = LT.first;
+ // When SVE2p1 or SME2 is available, we can halve getTypeLegalizationCost
+ // as get_active_lane_mask may lower to the sve_whilelo_x2 intrinsic, e.g.
+ // nxv32i1 = get_active_lane_mask(base, idx) ->
+ // {nxv16i1, nxv16i1} = sve_whilelo_x2(base, idx)
+ if (ST->hasSVE2p1() || ST->hasSME2()) {
+ Cost /= 2;
+ if (Cost == 1)
+ return Cost + ExtractCost;
+ }
+
+ // If more than one whilelo intrinsic is required, include the extra cost
+ // required by the saturating add & select required to increment the
+ // start value after the first intrinsic call.
+ IntrinsicCostAttributes AddAttrs(Intrinsic::uadd_sat, ArgTy,
+ {ArgTy, ArgTy});
+ InstructionCost SplitCost = getIntrinsicInstrCost(AddAttrs, CostKind);
+ Type *CondTy = ArgTy->getWithNewBitWidth(1);
+ SplitCost += getCmpSelInstrCost(Instruction::Select, ArgTy, CondTy,
+ CmpInst::ICMP_UGT, CostKind);
+ return (Cost + (SplitCost * (Cost - 1))) + ExtractCost;
+ } else if (!getTLI()->isTypeLegal(RetVT)) {
+ // We don't have enough context at this point to determine if the mask
+ // is going to be kept live after the block, which will force the vXi1
+ // type to be expanded to legal vectors of integers, e.g. v4i1->v4i32.
+ // For now, we just assume the vectorizer created this intrinsic and
+ // the result will be the input for a PHI. In this case the cost will
+ // be extremely high for fixed-width vectors.
+ // NOTE: getScalarizationOverhead returns a cost that's far too
+ // pessimistic for the actual generated codegen. In reality there are
+ // two instructions generated per lane.
+ return (cast<FixedVectorType>(ResTy)->getNumElements() * 2) + ExtractCost;
+ }
+
+ IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, ResTy, {ArgTy},
+ FMF);
+ return BaseT::getIntrinsicInstrCost(Attrs, CostKind) + ExtractCost;
+}
+
InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const {
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 9997686b33451..7284dbbe1065e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -246,6 +246,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
TTI::TargetCostKind CostKind) const override;
+ InstructionCost getActiveLaneMaskCost(Type *ResTy, Type *ArgTy,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const override;
+
InstructionCost
getArithmeticReductionCostSVE(unsigned Opcode, VectorType *ValTy,
TTI::TargetCostKind CostKind) const;
@@ -477,11 +482,6 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override;
- bool preferWideActiveLaneMasks() const override {
- return ST->isSVEorStreamingSVEAvailable() &&
- (ST->hasSVE2p1() || ST->hasSME2());
- }
-
bool supportsScalableVectors() const override {
return ST->isSVEorStreamingSVEAvailable();
}
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index c1df7fbb9d702..175a0c2ae2a8f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2084,22 +2084,28 @@ ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
}
+InstructionCost ARMTTIImpl::getActiveLaneMaskCost(Type *RetTy, Type *ArgTy,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const {
+ // Currently we make a somewhat optimistic assumption that
+ // active_lane_mask's are always free. In reality it may be freely folded
+ // into a tail predicated loop, expanded into a VCPT or expanded into a lot
+ // of add/icmp code. We may need to improve this in the future, but being
+ // able to detect if it is free or not involves looking at a lot of other
+ // code. We currently assume that the vectorizer inserted these, and knew
+ // what it was doing in adding one.
+ if (ST->hasMVEIntegerOps())
+ return 0;
+
+ return BaseT::getActiveLaneMaskCost(RetTy, ArgTy, FMF, CostKind, NumResults);
+}
+
InstructionCost
ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const {
unsigned Opc = ICA.getID();
switch (Opc) {
- case Intrinsic::get_active_lane_mask:
- // Currently we make a somewhat optimistic assumption that
- // active_lane_mask's are always free. In reality it may be freely folded
- // into a tail predicated loop, expanded into a VCPT or expanded into a lot
- // of add/icmp code. We may need to improve this in the future, but being
- // able to detect if it is free or not involves looking at a lot of other
- // code. We currently assume that the vectorizer inserted these, and knew
- // what it was doing in adding one.
- if (ST->hasMVEIntegerOps())
- return 0;
- break;
case Intrinsic::sadd_sat:
case Intrinsic::ssub_sat:
case Intrinsic::uadd_sat:
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index e824839e39159..89b18d39a379a 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -409,6 +409,11 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
TTI::TargetCostKind CostKind) const override;
+ InstructionCost getActiveLaneMaskCost(Type *ResTy, Type *ArgTy,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const override;
+
InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index ca82f3e1a147b..b07301dcace20 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1655,21 +1655,6 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
case Intrinsic::masked_srem:
return getArithmeticInstrCost(Instruction::SRem, ICA.getReturnType(),
CostKind);
- case Intrinsic::get_active_lane_mask: {
- if (ST->hasVInstructions()) {
- Type *ExpRetTy = VectorType::get(
- ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount());
- auto LT = getTypeLegalizationCost(ExpRetTy);
-
- // vid.v v8 // considered hoisted
- // vsaddu.vx v8, v8, a0
- // vmsltu.vx v0, v8, a1
- return LT.first *
- getRISCVInstructionCost({RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
- LT.second, CostKind);
- }
- break;
- }
// TODO: add more intrinsic
case Intrinsic::stepvector: {
auto LT = getTypeLegalizationCost(RetTy);
@@ -2150,6 +2135,36 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
+InstructionCost
+RISCVTTIImpl::getActiveLaneMaskCost(Type *RetTy, Type *ArgTy, FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const {
+ if (!ST->hasVInstructions())
+ return BaseT::getActiveLaneMaskCost(RetTy, ArgTy, FMF, CostKind,
+ NumResults);
+
+ Type *ExpRetTy =
+ VectorType::get(ArgTy, cast<VectorType>(RetTy)->getElementCount());
+ auto LT = getTypeLegalizationCost(ExpRetTy);
+
+ InstructionCost ExtractCost = 0;
+ if (NumResults > 1) {
+ auto ResSubTy = VectorType::getOneNthElementsVectorType(
+ cast<VectorType>(RetTy), NumResults);
+ IntrinsicCostAttributes ICA(Intrinsic::vector_extract, ResSubTy,
+ {RetTy, ArgTy});
+ ExtractCost = getIntrinsicInstrCost(ICA, CostKind) * NumResults;
+ }
+
+ // vid.v v8 // considered hoisted
+ // vsaddu.vx v8, v8, a0
+ // vmsltu.vx v0, v8, a1
+ return LT.first *
+ getRISCVInstructionCost({RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
+ LT.second, CostKind) +
+ ExtractCost;
+}
+
InstructionCost
RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 477c419228a6b..c23fb0de2e433 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -224,6 +224,11 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
TTI::TargetCostKind CostKind) const override;
+ InstructionCost getActiveLaneMaskCost(Type *ResTy, Type *ArgTy,
+ FastMathFlags FMF,
+ TTI::TargetCostKind CostKind,
+ unsigned NumResults) const override;
+
std::optional<InstructionCost> getCombinedArithmeticInstructionCost(
unsigned ISDOpcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45f3bfe8e3f6f..b12a1cc9c0f31 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1241,10 +1241,8 @@ class LoopVectorizationCostModel {
case (WideActiveLaneMask::Disable):
return false;
case (WideActiveLaneMask::Force):
- return true;
case (WideActiveLaneMask::Default):
- return TTI.preferWideActiveLaneMasks() &&
- getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
+ return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
}
llvm_unreachable("invalid enum");
}
@@ -1326,6 +1324,8 @@ class LoopVectorizationCostModel {
/// trivially hoistable.
bool shouldConsiderInvariant(Value *Op);
+ bool shouldWidenActiveLaneMask(ElementCount VF, unsigned IC);
+
private:
unsigned NumPredStores = 0;
@@ -4664,6 +4664,33 @@ LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
return getWideningCost(I, VF);
}
+bool LoopVectorizationCostModel::shouldWidenActiveLaneMask(ElementCount VF,
+ unsigned IC) {
+ if (EnableWideActiveLaneMask.getValue() == WideActiveLaneMask::Force ||
+ ForceTargetInstructionCost.getNumOccurrences() > 0)
+ return true;
+
+ LLVMContext &Ctx = TheFunction->getContext();
+ Type *ArgTy = Type::getInt64Ty(Ctx);
+
+ // Compare the cost of one narrow mask per part vs one wide lane mask
+ // with extracts.
+ Type *ResTy = VectorType::get(Type::getInt1Ty(Ctx), VF);
+ InstructionCost ALMCost =
+ TTI.getActiveLaneMaskCost(ResTy, ArgTy, FastMathFlags(),
+ TTI::TCK_RecipThroughput, 1) *
+ IC;
+
+ ResTy = VectorType::get(Type::getInt1Ty(Ctx), VF * IC);
+ InstructionCost WideALMCost = TTI.getActiveLaneMaskCost(
+ ResTy, ArgTy, FastMathFlags(), TTI::TCK_RecipThroughput, IC);
+
+ if (WideALMCost == InstructionCost::getInvalid())
+ return false;
+
+ return WideALMCost < ALMCost;
+}
+
InstructionCost
LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
ElementCount VF) const {
@@ -8359,6 +8386,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Select the interleave count.
IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
+ if (IC > 1 && !CM.isEpilogueAllowed() &&
+ (CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()) &&
+ !CM.shouldWidenActiveLaneMask(VF.Width, IC))
+ IC = 1;
+
unsigned SelectedIC = std::max(IC, UserIC);
// Optimistically generate runtime checks if they are needed. Drop them if
// they turn out to not be profitable.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ef217b92640c4..5b348ca56dcfd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1224,9 +1224,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));
unsigned Multiplier = cast<VPConstantInt>(getOperand(2))->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
- IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
- {ArgTy, ArgTy});
- return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ return Ctx.TTI.getActiveLaneMaskCost(RetTy, ArgTy, FastMathFlags(),
+ Ctx.CostKind, Multiplier);
}
case VPInstruction::ExplicitVectorLength: {
Type *Arg0Ty = Ctx.Types.inferScalarType(getOperand(0));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 03b4c1a2ce458..93ffeee93a6c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2138,11 +2138,27 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
unsigned UF,
const TargetTransformInfo &TTI) {
if (EnableWideActiveLaneMask == WideActiveLaneMask::Disable ||
- (EnableWideActiveLaneMask == WideActiveLaneMask::Default &&
- !TTI.preferWideActiveLaneMasks()) ||
!VF.isVector() || UF == 1)
return false;
+ LLVMContext &Ctx = Plan.getContext();
+ if (EnableWideActiveLaneMask != WideActiveLaneMask::Force &&
+ ForceTargetInstructionCost.getNumOccurrences() == 0) {
+ Type *ArgTy = Type::getInt64Ty(Ctx);
+ Type *ResTy = VectorType::get(Type::getInt1Ty(Ctx), VF);
+ InstructionCost ALMCost =
+ TTI.getActiveLaneMaskCost(ResTy, ArgTy, FastMathFlags(),
+ TTI::TCK_RecipThroughput, 1) *
+ UF;
+
+ ResTy = VectorType::get(Type::getInt1Ty(Ctx), VF * UF);
+ InstructionCost WideALMCost = TTI.getActiveLaneMaskCost(
+ ResTy, ArgTy, FastMathFlags(), TTI::TCK_RecipThroughput, UF);
+
+ if (WideALMCost == InstructionCost::getInvalid() || WideALMCost >= ALMCost)
+ return false;
+ }
+
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
auto *Term = &ExitingVPBB->back();
@@ -2153,8 +2169,6 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
return false;
auto *Header = cast<VPBasicBlock>(VectorRegion->getEntry());
- LLVMContext &Ctx = Plan.getContext();
-
auto ExtractFromALM = [&](VPInstruction *ALM,
SmallVectorImpl<VPValue *> &Extracts) {
DebugLoc DL = ALM->getDebugLoc();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 2cd9781bd18a7..6dae71ad60957 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -40,6 +40,7 @@ struct VFRange;
LLVM_ABI_FOR_TEST extern cl::opt<bool> VerifyEachVPlan;
LLVM_ABI_FOR_TEST extern cl::opt<WideActiveLaneMask> EnableWideActiveLaneMask;
+LLVM_ABI_FOR_TEST extern cl::opt<unsigned> ForceTargetInstructionCost;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_ABI_FOR_TEST extern cl::opt<bool> VPlanPrintAfterAll;
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 2a81448db1143..6eb82d1fcea6d 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -927,10 +927,12 @@ define void @get_lane_mask() #0 {
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 poison, i64 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 poison, i64 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 poison, i64 poison)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i64 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64 poison, i64 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 poison, i32 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 poison, i32 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 poison, i32 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 poison, i32 poison)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i32 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32 poison, i32 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 13 for: %mask_nxv64i1_i64 = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 poison, i64 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 5 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 poison, i64 poison)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 poison, i16 poison)
@@ -951,10 +953,12 @@ define void @get_lane_mask() #0 {
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 poison, i64 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 poison, i64 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 poison, i64 poison)
+; CHECK-SVE-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i64 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64 poison, i64 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 poison, i32 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 poison, i32 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 poison, i32 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 poison, i32 poison)
+; CHECK-SVE-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i32 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32 poison, i32 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 13 for: %mask_nxv64i1_i64 = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 poison, i64 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 5 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 poison, i64 poison)
; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 poison, i16 poison)
@@ -975,10 +979,12 @@ define void @get_lane_mask() #0 {
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 poison, i64 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 poison, i64 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 poison, i64 poison)
+; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i64 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64 poison, i64 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 poison, i32 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 poison, i32 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 poison, i32 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 poison, i32 poison)
+; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i32 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32 poison, i32 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 5 for: %mask_nxv64i1_i64 = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 poison, i64 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 poison, i64 poison)
; CHECK-SVE2p1-OR-SME2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 poison, i16 poison)
@@ -999,10 +1005,12 @@ define void @get_lane_mask() #0 {
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 poison, i64 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 poison, i64 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 poison, i64 poison)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i64 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64 poison, i64 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 poison, i32 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 poison, i32 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 poison, i32 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 poison, i32 poison)
+; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of Invalid for: %mask_nxv1i1_i32 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32 poison, i32 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 13 for: %mask_nxv64i1_i64 = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 poison, i64 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 5 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 poison, i64 poison)
; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 poison, i16 poison)
@@ -1022,11 +1030,13 @@ define void @get_lane_mask() #0 {
%mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 poison, i64 poison)
%mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 poison, i64 poison)
%mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 poison, i64 poison)
+ %mask_nxv1i1_i64 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i64(i64 poison, i64 poison)
%mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 poison, i32 poison)
%mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 poison, i32 poison)
%mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 poison, i32 poison)
%mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 poison, i32 poison)
+ %mask_nxv1i1_i32 = call <vscale x 1 x i1> @llvm.get.active.lane.mask.nxv1i1.i32(i32 poison, i32 poison)
%mask_nxv64i1_i64 = call <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64 poison, i64 poison)
%mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 poison, i64 poison)
@@ -1751,45 +1761,6 @@ define void @abs() #1 {
ret void
}
-
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64, i64)
-declare <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64, i64)
-declare <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64, i64)
-declare <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64, i64)
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32, i32)
-declare <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32, i32)
-declare <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32, i32)
-declare <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32, i32)
-declare <vscale x 64 x i1> @llvm.get.active.lane.mask.nxv64i1.i64(i64, i64)
-declare <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64, i64)
-declare <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16, i16)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64, i64)
-declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64, i64)
-declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64, i64)
-declare <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64, i64)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)
-declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)
-declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
-declare <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32, i32)
-declare <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64, i64)
-declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16, i16)
-declare <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
-declare <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
-declare <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
-declare <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 %align, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
-declare <vscale x 8 x i32> @llvm.masked.gather.nxv8i32(<vscale x 8 x ptr> %ptrs, i32 %align, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
-declare <4 x i32> @llvm.masked.gather.v4i32(<4 x ptr> %ptrs, i32 %align, <4 x i1> %masks, <4 x i32> %passthru)
-declare <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr>, i32, <1 x i1>, <1 x i128>)
-declare void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 %align, <vscale x 4 x i1> %masks)
-declare void @llvm.masked.scatter.nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 %align, <vscale x 8 x i1> %masks)
-declare void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x ptr> %ptrs, i32 %align, <4 x i1> %masks)
-declare void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 %align, <1 x i1> %masks)
-
attributes #0 = { "target-features"="+sve,+bf16" }
attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
attributes #2 = { "target-features"="+sve" vscale_range(2, 16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
index 6418bc21733a0..f73ebf315fbe9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
@@ -6,6 +6,8 @@
; RUN: -enable-wide-lane-mask=force -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
; RUN: -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=default -mattr=+sve -force-target-instruction-cost=1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
; RUN: -enable-wide-lane-mask=default -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-NO-WIDEALM
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
index 37bb827f893a1..30f6047f5b72c 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
@@ -1,5 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=disable \
+; RUN: -force-tail-folding-style=data-and-control -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=default \
+; RUN: -force-tail-folding-style=data-and-control -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force \
+; RUN: -force-tail-folding-style=data-and-control -S | FileCheck %s --check-prefix=CHECK-WIDE-ALM
target triple = "thumbv8.1m.main-arm-unknown-eabihf"
@@ -12,15 +17,13 @@ define void @f0(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 31
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 32
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 16, i64 [[N]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX]], i64 [[N]])
-; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[TMP0]], i64 [[N]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
@@ -32,8 +35,12 @@ define void @f0(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP4]], ptr align 1 [[TMP6]], <16 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP5]], ptr align 1 [[TMP8]], <16 x i1> [[ACTIVE_LANE_MASK1]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[TMP9]], i64 [[N]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP7]], true
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; CHECK: [[FOR_END_LOOPEXIT]]:
@@ -41,6 +48,46 @@ define void @f0(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
+; CHECK-WIDE-ALM-LABEL: define void @f0(
+; CHECK-WIDE-ALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-WIDE-ALM-NEXT: [[ENTRY:.*:]]
+; CHECK-WIDE-ALM-NEXT: [[VAL:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-WIDE-ALM-NEXT: br i1 [[VAL]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; CHECK-WIDE-ALM: [[FOR_BODY_PREHEADER]]:
+; CHECK-WIDE-ALM-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-WIDE-ALM: [[VECTOR_PH]]:
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[N]])
+; CHECK-WIDE-ALM-NEXT: [[TMP0:%.*]] = call <16 x i1> @llvm.vector.extract.v16i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; CHECK-WIDE-ALM-NEXT: [[TMP1:%.*]] = call <16 x i1> @llvm.vector.extract.v16i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-WIDE-ALM-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-WIDE-ALM: [[VECTOR_BODY]]:
+; CHECK-WIDE-ALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <16 x i1> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <16 x i1> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-WIDE-ALM-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 16
+; CHECK-WIDE-ALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; CHECK-WIDE-ALM-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[ACTIVE_LANE_MASK1]], <16 x i8> poison)
+; CHECK-WIDE-ALM-NEXT: [[TMP4:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
+; CHECK-WIDE-ALM-NEXT: [[TMP5:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD2]], splat (i8 3)
+; CHECK-WIDE-ALM-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-WIDE-ALM-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 16
+; CHECK-WIDE-ALM-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP4]], ptr align 1 [[TMP6]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-WIDE-ALM-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP5]], ptr align 1 [[TMP7]], <16 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-WIDE-ALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-WIDE-ALM-NEXT: [[TMP8]] = call <16 x i1> @llvm.vector.extract.v16i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
+; CHECK-WIDE-ALM-NEXT: [[TMP9]] = call <16 x i1> @llvm.vector.extract.v16i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-WIDE-ALM-NEXT: [[TMP10:%.*]] = extractelement <16 x i1> [[TMP9]], i64 0
+; CHECK-WIDE-ALM-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-WIDE-ALM-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-WIDE-ALM: [[MIDDLE_BLOCK]]:
+; CHECK-WIDE-ALM-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
+; CHECK-WIDE-ALM: [[FOR_END_LOOPEXIT]]:
+; CHECK-WIDE-ALM-NEXT: br label %[[FOR_END]]
+; CHECK-WIDE-ALM: [[FOR_END]]:
+; CHECK-WIDE-ALM-NEXT: ret void
+;
entry:
%val = icmp sgt i64 %n, 0
br i1 %val, label %for.body, label %for.end
@@ -70,3 +117,7 @@ attributes #0 = { nofree "target-features"="+armv8.1-m.main,+mve.fp" }
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
;.
+; CHECK-WIDE-ALM: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-WIDE-ALM: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-WIDE-ALM: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/active-lane-mask.ll
new file mode 100644
index 0000000000000..b817e9d050643
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/active-lane-mask.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-tail-folding-style=data-and-control \
+; RUN: -enable-wide-lane-mask=disable -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-ALM
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-tail-folding-style=data-and-control \
+; RUN: -enable-wide-lane-mask=default -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-ALM
+; RUN: opt -mtriple=riscv64-none-linux-gnu -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-tail-folding-style=data-and-control \
+; RUN: -enable-wide-lane-mask=force -force-vector-interleave=2 < %s | FileCheck %s -check-prefix CHECK-WIDE-ALM
+
+define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-ALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-ALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-ALM-NEXT: entry:
+; CHECK-ALM-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-ALM: vector.ph:
+; CHECK-ALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-ALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP1]], i64 [[N]])
+; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-ALM: vector.body:
+; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ALM-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-ALM-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 [[TMP1]]
+; CHECK-ALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-ALM-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 4 x i32> poison)
+; CHECK-ALM-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 3)
+; CHECK-ALM-NEXT: [[TMP6:%.*]] = mul <vscale x 4 x i32> [[WIDE_MASKED_LOAD3]], splat (i32 3)
+; CHECK-ALM-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-ALM-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 [[TMP1]]
+; CHECK-ALM-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP5]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-ALM-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP8]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK2]])
+; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-ALM-NEXT: [[TMP9:%.*]] = add i64 [[INDEX_NEXT]], [[TMP1]]
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP9]], i64 [[N]])
+; CHECK-ALM-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-ALM-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-ALM-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-ALM: middle.block:
+;
+; CHECK-WIDE-ALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-WIDE-ALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-WIDE-ALM-NEXT: entry:
+; CHECK-WIDE-ALM-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-WIDE-ALM: vector.ph:
+; CHECK-WIDE-ALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-WIDE-ALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-WIDE-ALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
+; CHECK-WIDE-ALM-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
+; CHECK-WIDE-ALM-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-WIDE-ALM-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-WIDE-ALM: vector.body:
+; CHECK-WIDE-ALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-WIDE-ALM-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-WIDE-ALM-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP1]]
+; CHECK-WIDE-ALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-WIDE-ALM-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 4 x i32> poison)
+; CHECK-WIDE-ALM-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 3)
+; CHECK-WIDE-ALM-NEXT: [[TMP8:%.*]] = mul <vscale x 4 x i32> [[WIDE_MASKED_LOAD2]], splat (i32 3)
+; CHECK-WIDE-ALM-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-WIDE-ALM-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP1]]
+; CHECK-WIDE-ALM-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-WIDE-ALM-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP10]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-WIDE-ALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-WIDE-ALM-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-WIDE-ALM-NEXT: [[TMP11]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
+; CHECK-WIDE-ALM-NEXT: [[TMP12]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-WIDE-ALM-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[TMP12]], i64 0
+; CHECK-WIDE-ALM-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-WIDE-ALM-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-WIDE-ALM: middle.block:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx1 = getelementptr inbounds i32, ptr %src, i64 %iv
+ %ld = load i32, ptr %arrayidx1
+ %mul = mul i32 %ld, 3
+ %arrayidx2 = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %mul, ptr %arrayidx2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+attributes #0 = { "target-features"="+v" }
More information about the llvm-commits
mailing list