[llvm] [LV] Enable wide active lane masks when tail-folding if preferred (PR #193757)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 07:05:34 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Kerry McLaughlin (kmclaughlin-arm)
<details>
<summary>Changes</summary>
Adds a new TTI hook which returns true if the target prefers wide
active lane masks. For AArch64 this returns true for targets with
SVE2p1 or SME2, as this means the whilelo (predicate pair)
instruction is available.
When preferWideActiveLaneMasks returns true, wide active lane masks
will be considered when tail-folding is also enabled.
This does not affect the default behaviour of any target.
---
Patch is 23.16 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/193757.diff
11 Files Affected:
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+12)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+2)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+4)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+5)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+21-7)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+9-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+4-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll (+4-4)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll (+121)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll (+1-1)
``````````diff
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 333c5e4868395..683eaa4133b80 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -253,6 +253,16 @@ enum class TailFoldingStyle {
DataWithEVL,
};
+enum class WideActiveLaneMask {
+ // Do not consider using wide active lane masks.
+ Disable,
+ // Considered when the TailFoldingStyle is DataAndControlFlow and
+ // preferWideActiveLaneMasks() returns true for the target.
+ Default,
+ // Always consider using wide active lane masks.
+ Force,
+};
+
struct TailFoldingInfo {
TargetLibraryInfo *TLI;
LoopVectorizationLegality *LVL;
@@ -1961,6 +1971,8 @@ class TargetTransformInfo {
/// processing \p Iters scalar iterations per vector iteration.
LLVM_ABI bool preferEpilogueVectorization(ElementCount Iters) const;
+ LLVM_ABI bool preferWideActiveLaneMasks() const;
+
/// \returns True if the loop vectorizer should discard any VFs where the
/// maximum register pressure exceeds getNumberOfRegisters.
LLVM_ABI bool shouldConsiderVectorizationRegPressure() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 49349a22e21e9..dfd14de1003df 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1163,6 +1163,8 @@ class TargetTransformInfoImplBase {
return getMaxInterleaveFactor(Iters) > 1;
}
+ virtual bool preferWideActiveLaneMasks() const { return false; }
+
virtual bool shouldConsiderVectorizationRegPressure() const { return false; }
virtual bool shouldExpandReduction(const IntrinsicInst *II) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index e1ab90a8e046c..74207de53139f 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1481,6 +1481,10 @@ bool TargetTransformInfo::preferEpilogueVectorization(
return TTIImpl->preferEpilogueVectorization(Iters);
}
+bool TargetTransformInfo::preferWideActiveLaneMasks() const {
+ return TTIImpl->preferWideActiveLaneMasks();
+}
+
bool TargetTransformInfo::shouldConsiderVectorizationRegPressure() const {
return TTIImpl->shouldConsiderVectorizationRegPressure();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 8125b7ffcae43..9997686b33451 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -477,6 +477,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override;
+ bool preferWideActiveLaneMasks() const override {
+ return ST->isSVEorStreamingSVEAvailable() &&
+ (ST->hasSVE2p1() || ST->hasSME2());
+ }
+
bool supportsScalableVectors() const override {
return ST->isSVEorStreamingSVEAvailable();
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6f079062a6e1a..8c22d4bea8039 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -243,10 +243,18 @@ static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"Use predicated EVL instructions for tail folding. If EVL "
"is unsupported, fallback to data-without-lane-mask.")));
-cl::opt<bool> llvm::EnableWideActiveLaneMask(
- "enable-wide-lane-mask", cl::init(false), cl::Hidden,
+cl::opt<WideActiveLaneMask> llvm::EnableWideActiveLaneMask(
+ "enable-wide-lane-mask",
cl::desc("Enable use of wide lane masks when used for control flow in "
- "tail-folded loops"));
+ "tail-folded loops"),
+ cl::init(WideActiveLaneMask::Default),
+ cl::values(clEnumValN(WideActiveLaneMask::Default, "default",
+ "Decision to use wide active lane masks based on "
+ "target preference."),
+ clEnumValN(WideActiveLaneMask::Disable, "disable",
+ "Use of wide active lane masks disabled."),
+ clEnumValN(WideActiveLaneMask::Force, "force",
+ "Always use wide active lane masks where possible")));
static cl::opt<bool> MaximizeBandwidth(
"vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
@@ -1353,10 +1361,16 @@ class LoopVectorizationCostModel {
/// Returns true if the use of wide lane masks is requested and the loop is
/// using tail-folding with a lane mask for control flow.
bool useWideActiveLaneMask() const {
- if (!EnableWideActiveLaneMask)
+ switch (EnableWideActiveLaneMask.getValue()) {
+ case (WideActiveLaneMask::Disable):
return false;
-
- return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
+ case (WideActiveLaneMask::Force):
+ return true;
+ case (WideActiveLaneMask::Default):
+ return TTI.preferWideActiveLaneMasks() &&
+ getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
+ }
+ llvm_unreachable("invalid enum");
}
/// Return maximum safe number of elements to be processed per vector
@@ -6984,7 +6998,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::materializeConstantVectorTripCount(BestVPlan, BestVF, BestUF,
PSE);
- VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
+ VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE, TTI);
VPlanTransforms::simplifyRecipes(BestVPlan);
if (EpilogueVecKind == EpilogueVectorizationKind::None)
VPlanTransforms::removeBranchOnConst(BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 406556ab70993..ef185f165a441 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2118,8 +2118,12 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
/// new extracts from the first active lane mask, which has it's last
/// operand (multiplier) set to UF.
static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
- unsigned UF) {
- if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1)
+ unsigned UF,
+ const TargetTransformInfo &TTI) {
+ if (EnableWideActiveLaneMask == WideActiveLaneMask::Disable ||
+ (EnableWideActiveLaneMask == WideActiveLaneMask::Default &&
+ !TTI.preferWideActiveLaneMasks()) ||
+ !VF.isVector() || UF == 1)
return false;
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -2304,11 +2308,12 @@ bool VPlanTransforms::simplifyKnownEVL(VPlan &Plan, ElementCount VF,
void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
unsigned BestUF,
- PredicatedScalarEvolution &PSE) {
+ PredicatedScalarEvolution &PSE,
+ const TargetTransformInfo &TTI) {
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
- bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF);
+ bool MadeChange = tryToReplaceALMWithWideALM(Plan, BestVF, BestUF, TTI);
MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 28ccab704c790..cbde54c0cbdf9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -17,6 +17,7 @@
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Regex.h"
@@ -38,7 +39,7 @@ class VPRecipeBuilder;
struct VFRange;
LLVM_ABI_FOR_TEST extern cl::opt<bool> VerifyEachVPlan;
-LLVM_ABI_FOR_TEST extern cl::opt<bool> EnableWideActiveLaneMask;
+LLVM_ABI_FOR_TEST extern cl::opt<WideActiveLaneMask> EnableWideActiveLaneMask;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_ABI_FOR_TEST extern cl::opt<bool> VPlanPrintAfterAll;
@@ -252,7 +253,8 @@ struct VPlanTransforms {
/// resulting plan to \p BestVF and \p BestUF.
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
unsigned BestUF,
- PredicatedScalarEvolution &PSE);
+ PredicatedScalarEvolution &PSE,
+ const TargetTransformInfo &TTI);
/// Try to simplify VPInstruction::ExplicitVectorLength recipes when the AVL
/// is known to be <= VF, replacing them with the AVL directly.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index 2be6f1db3a1c5..cd4704066dfd9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force \
; RUN: -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask \
+; RUN: opt -S --passes=loop-vectorize -scalable-vectorization=off -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force \
; RUN: -force-vector-width=4 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
target triple = "aarch64-unknown-linux"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
index 85ebfdb7132b9..5e187b35bd7db 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-wide-lane-mask.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 4
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
-; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
-; RUN: opt -S --passes=loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail < %s | FileCheck %s -check-prefix CHECK-TF
-; RUN: opt -S --passes=forceattrs,loop-vectorize -enable-wide-lane-mask -tail-folding-policy=must-fold-tail -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-vector-interleave=1 < %s | FileCheck %s -check-prefix CHECK-UF1
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-vector-interleave=4 < %s | FileCheck %s -check-prefix CHECK-UF4
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force < %s | FileCheck %s -check-prefix CHECK-TF
+; RUN: opt -S --passes=forceattrs,loop-vectorize -tail-folding-policy=must-fold-tail -enable-wide-lane-mask=force -force-attribute=optsize < %s | FileCheck %s -check-prefix CHECK-UF1
target triple = "aarch64-unknown-linux"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
new file mode 100644
index 0000000000000..be224fe1c9a7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-lane-mask-flags.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^middle.block:" --version 6
+
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=default -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=force -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-WIDEALM
+
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=default -mattr=+sve < %s | FileCheck %s -check-prefix=CHECK-NO-WIDEALM
+; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-interleave=2 \
+; RUN: -enable-wide-lane-mask=disable -mattr=+sve2p1 < %s | FileCheck %s -check-prefix=CHECK-NO-WIDEALM
+
+target triple = "aarch64-unknown-linux"
+
+define void @scalable_wide_active_lane_mask(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
+; CHECK-WIDEALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-WIDEALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-WIDEALM-NEXT: [[ENTRY:.*:]]
+; CHECK-WIDEALM-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-WIDEALM-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-WIDEALM: [[FOR_BODY_PREHEADER]]:
+; CHECK-WIDEALM-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-WIDEALM: [[VECTOR_PH]]:
+; CHECK-WIDEALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-WIDEALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-WIDEALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-WIDEALM-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; CHECK-WIDEALM-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-WIDEALM-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-WIDEALM: [[VECTOR_BODY]]:
+; CHECK-WIDEALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 2 x i1> [ [[TMP3]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-WIDEALM-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-WIDEALM-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 [[TMP1]]
+; CHECK-WIDEALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-WIDEALM-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]], <vscale x 2 x double> poison)
+; CHECK-WIDEALM-NEXT: [[TMP7:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-WIDEALM-NEXT: [[TMP8:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD2]], splat (double 3.000000e+00)
+; CHECK-WIDEALM-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-WIDEALM-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP1]]
+; CHECK-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP7]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-WIDEALM-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK1]])
+; CHECK-WIDEALM-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-WIDEALM-NEXT: [[TMP11]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 2)
+; CHECK-WIDEALM-NEXT: [[TMP12]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-WIDEALM-NEXT: [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; CHECK-WIDEALM-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-WIDEALM-NEXT: br i1 [[TMP14]], label %[[FOR_END_LOOPEXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-WIDEALM: [[FOR_END_LOOPEXIT]]:
+;
+; CHECK-NO-WIDEALM-LABEL: define void @scalable_wide_active_lane_mask(
+; CHECK-NO-WIDEALM-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NO-WIDEALM-NEXT: [[ENTRY:.*:]]
+; CHECK-NO-WIDEALM-NEXT: [[CMP6:%.*]] = icmp sgt i64 [[N]], 0
+; CHECK-NO-WIDEALM-NEXT: br i1 [[CMP6]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-NO-WIDEALM: [[FOR_BODY_PREHEADER]]:
+; CHECK-NO-WIDEALM-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NO-WIDEALM: [[VECTOR_PH]]:
+; CHECK-NO-WIDEALM-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NO-WIDEALM-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NO-WIDEALM-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP1]], i64 [[N]])
+; CHECK-NO-WIDEALM-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NO-WIDEALM: [[VECTOR_BODY]]:
+; CHECK-NO-WIDEALM-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NO-WIDEALM-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NO-WIDEALM-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 [[TMP1]]
+; CHECK-NO-WIDEALM-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-NO-WIDEALM-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
+; CHECK-NO-WIDEALM-NEXT: [[TMP5:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 3.000000e+00)
+; CHECK-NO-WIDEALM-NEXT: [[TMP6:%.*]] = fmul <vscale x 2 x double> [[...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/193757
More information about the llvm-commits
mailing list