[llvm] 2c1ca89 - [LLVM][CodeGen][AArch64] Implement expansion for fixed-length get.active.lane.mask. (#225459)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 07:14:21 PDT 2026
Author: Paul Walker
Date: 2026-09-24T15:14:14+01:00
New Revision: 2c1ca898c2baed18179811e088f7694b1927a816
URL: https://github.com/llvm/llvm-project/commit/2c1ca898c2baed18179811e088f7694b1927a816
DIFF: https://github.com/llvm/llvm-project/commit/2c1ca898c2baed18179811e088f7694b1927a816.diff
LOG: [LLVM][CodeGen][AArch64] Implement expansion for fixed-length get.active.lane.mask. (#225459)
A step in the direction of removing the shouldExpandGetActiveLaneMask
TTI hook. For this PR I've implemented enough functionality to allow all
legalish fixed-length variants of get.active.lane.mask to bypass the
SelectionDAGBuilder based expansion for AArch64.
The AArch64TargetLowering::shouldExpandGetActiveLaneMask restrictions
are related to maintaining current code quality and not because the
expansion code cannot handle them.
Tests are a copy of active_lane_mask.ll with the scalable vector tests
removed.
Added:
llvm/test/CodeGen/AArch64/neon-get-active-lane-mask.ll
Modified:
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
llvm/lib/CodeGen/TargetLoweringBase.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/Analysis/CostModel/AArch64/loop_dependence_mask.ll
llvm/test/CodeGen/AArch64/get-active-lane-mask-extract-nosve.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 5e3f252fdd3d4..edb8a971eef2b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -135,6 +135,7 @@ class VectorLegalizer {
SDValue ExpandVSELECT(SDNode *Node);
SDValue ExpandVP_MERGE(SDNode *Node);
SDValue ExpandVP_REM(SDNode *Node);
+ SDValue ExpandGET_ACTIVE_LANE_MASK(SDNode *N);
SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
SDValue ExpandMaskedBinOp(SDNode *N);
SDValue ExpandSELECT(SDNode *Node);
@@ -481,6 +482,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::VECTOR_COMPRESS:
case ISD::SCMP:
case ISD::UCMP:
+ case ISD::GET_ACTIVE_LANE_MASK:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
case ISD::LOOP_DEPENDENCE_RAW_MASK:
case ISD::MASKED_UDIV:
@@ -1314,6 +1316,10 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::UCMP:
Results.push_back(TLI.expandCMP(Node, DAG));
return;
+ case ISD::GET_ACTIVE_LANE_MASK:
+ if (SDValue R = ExpandGET_ACTIVE_LANE_MASK(Node))
+ Results.push_back(R);
+ return;
case ISD::LOOP_DEPENDENCE_WAR_MASK:
case ISD::LOOP_DEPENDENCE_RAW_MASK:
Results.push_back(ExpandLOOP_DEPENDENCE_MASK(Node));
@@ -1737,6 +1743,47 @@ SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) {
return DAG.getNode(ISD::SUB, DL, VT, Dividend, Mul);
}
+SDValue VectorLegalizer::ExpandGET_ACTIVE_LANE_MASK(SDNode *N) {
+ SDLoc DL(N);
+
+ SDValue Start = N->getOperand(0);
+ SDValue End = N->getOperand(1);
+ EVT VT = N->getValueType(0);
+ EVT OpVT = Start.getValueType();
+
+ if (VT.isScalableVector())
+ return SDValue();
+
+ // Try a promoted comparison type to simplify saturation.
+ EVT PromoteVT = VT.changeVectorElementType(*DAG.getContext(), OpVT);
+ if (TLI.isTypeLegal(PromoteVT) &&
+ isUIntN(OpVT.getScalarSizeInBits(), VT.getVectorNumElements())) {
+ SDValue StartV = DAG.getSplat(PromoteVT, DL, Start);
+ SDValue Seq = DAG.getStepVector(DL, PromoteVT);
+ Seq = DAG.getNode(ISD::UADDSAT, DL, PromoteVT, Seq, StartV);
+
+ EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ PromoteVT);
+ SDValue EndV = DAG.getSplat(PromoteVT, DL, End);
+ SDValue Mask = DAG.getSetCC(DL, MaskVT, Seq, EndV, ISD::SETULT);
+ return DAG.getBoolExtOrTrunc(Mask, DL, VT, PromoteVT);
+ }
+
+ // Is VT's element type big enough to hold all rebased indices?
+ if (!isUIntN(VT.getScalarSizeInBits(), VT.getVectorNumElements()))
+ return SDValue();
+
+ // Rebase and saturate the termination value.
+ SDValue Max = DAG.getConstant(maxUIntN(VT.getScalarSizeInBits()), DL, OpVT);
+ End = DAG.getNode(ISD::USUBSAT, DL, OpVT, End, Start);
+ End = DAG.getNode(ISD::UMIN, DL, OpVT, End, Max);
+
+ // cmp <0, 1, 2, 3...>, End
+ SDValue EndV = DAG.getSplat(VT, DL, End);
+ SDValue StepVector = DAG.getStepVector(DL, VT);
+ return DAG.getSetCC(DL, VT, StepVector, EndV, ISD::SETULT);
+}
+
SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
return TLI.expandLoopDependenceMask(N, DAG);
}
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f852f7551ce16..79badc63c62e1 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -952,10 +952,9 @@ void TargetLoweringBase::initActions() {
// Only some target support these vector operations. Default them to Expand.
setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
-
- // cttz.elts defaults to expand.
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
Expand);
+ setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Expand);
// VP operations default to expand.
#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d5135b47f4ef1..d048d0eb7acf4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2542,10 +2542,14 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
EVT OpVT) const {
- // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
- if (!Subtarget->isSVEorStreamingSVEAvailable() ||
- ResVT.getVectorElementType() != MVT::i1)
- return true;
+ if (!Subtarget->isSVEorStreamingSVEAvailable() &&
+ ResVT.isFixedLengthVector()) {
+ // Without SVE support only allow promotable result types.
+ if (!is_contained({2u, 4u, 8u, 16u}, ResVT.getVectorNumElements()))
+ return true;
+ if (OpVT != MVT::i32 && OpVT != MVT::i64)
+ return true;
+ }
// Expand 1 length fixed length vector.
if (ResVT.isFixedLengthVector() && ResVT.getVectorNumElements() == 1)
@@ -16979,6 +16983,8 @@ static SDValue NormalizeBuildVector(SDValue Op,
} else if (Lane.getOpcode() == ISD::UNDEF) {
Lane = DAG.getUNDEF(MVT::i32);
} else {
+ if (Lane.getValueType() == MVT::i64)
+ Lane = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Lane);
assert(Lane.getValueType() == MVT::i32 &&
"Unexpected BUILD_VECTOR operand type");
}
@@ -17374,6 +17380,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
if (!isConstant) {
LLVM_DEBUG(
dbgs() << "LowerBUILD_VECTOR: use DUP for non-constant splats\n");
+ if (Value.getValueType() == MVT::i64 && VT.getScalarSizeInBits() <= 32)
+ Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Value);
return DAG.getNode(AArch64ISD::DUP, DL, VT, Value);
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/loop_dependence_mask.ll b/llvm/test/Analysis/CostModel/AArch64/loop_dependence_mask.ll
index 6d0df7e184447..7ff94295235d6 100644
--- a/llvm/test/Analysis/CostModel/AArch64/loop_dependence_mask.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/loop_dependence_mask.ll
@@ -106,11 +106,11 @@ define void @loop_dependence_war_mask_invalid.i64(i64 %a, i64 %b) {
; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-SME-LABEL: 'loop_dependence_war_mask_invalid.i64'
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 2)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 1)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %res5 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 10)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.war.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.war.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 %a, i64 %b, i64 2)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 1)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %res5 = call <2 x i1> @llvm.loop.dependence.war.mask.v2i1.i64(i64 %a, i64 %b, i64 10)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res6 = call <vscale x 16 x i1> @llvm.loop.dependence.war.mask.nxv16i1.i64(i64 %a, i64 %b, i64 8)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res7 = call <vscale x 8 x i1> @llvm.loop.dependence.war.mask.nxv8i1.i64(i64 %a, i64 %b, i64 4)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res8 = call <vscale x 4 x i1> @llvm.loop.dependence.war.mask.nxv4i1.i64(i64 %a, i64 %b, i64 2)
@@ -161,11 +161,11 @@ define void @loop_dependence_raw_mask_invalid.i64(i64 %a, i64 %b) {
; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-SME-LABEL: 'loop_dependence_raw_mask_invalid.i64'
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1.i64(i64 %a, i64 %b, i64 2)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 1)
-; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %res5 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 10)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 41 for instruction: %res1 = call <16 x i1> @llvm.loop.dependence.raw.mask.v16i1.i64(i64 %a, i64 %b, i64 8)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %res2 = call <8 x i1> @llvm.loop.dependence.raw.mask.v8i1.i64(i64 %a, i64 %b, i64 4)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %res3 = call <4 x i1> @llvm.loop.dependence.raw.mask.v4i1.i64(i64 %a, i64 %b, i64 2)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %res4 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 1)
+; CHECK-SME-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %res5 = call <2 x i1> @llvm.loop.dependence.raw.mask.v2i1.i64(i64 %a, i64 %b, i64 10)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res6 = call <vscale x 16 x i1> @llvm.loop.dependence.raw.mask.nxv16i1.i64(i64 %a, i64 %b, i64 8)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res7 = call <vscale x 8 x i1> @llvm.loop.dependence.raw.mask.nxv8i1.i64(i64 %a, i64 %b, i64 4)
; CHECK-SME-NEXT: Cost Model: Invalid cost for instruction: %res8 = call <vscale x 4 x i1> @llvm.loop.dependence.raw.mask.nxv4i1.i64(i64 %a, i64 %b, i64 2)
diff --git a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract-nosve.ll b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract-nosve.ll
index 993d45a5a126c..cc6626f3725bf 100644
--- a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract-nosve.ll
+++ b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract-nosve.ll
@@ -5,20 +5,19 @@ target triple = "aarch64-linux"
define void @test_fixed_extract(i64 %i, i64 %n) #0 {
; CHECK-LABEL: test_fixed_extract:
; CHECK: // %bb.0:
-; CHECK-NEXT: adrp x8, .LCPI0_1
-; CHECK-NEXT: dup v0.2d, x0
-; CHECK-NEXT: adrp x9, .LCPI0_0
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_1]
-; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI0_0]
-; CHECK-NEXT: uqadd v1.2d, v0.2d, v1.2d
-; CHECK-NEXT: uqadd v0.2d, v0.2d, v2.2d
-; CHECK-NEXT: dup v2.2d, x1
-; CHECK-NEXT: cmhi v1.2d, v2.2d, v1.2d
-; CHECK-NEXT: cmhi v0.2d, v2.2d, v0.2d
-; CHECK-NEXT: xtn v1.2s, v1.2d
-; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: // fake_use: $d1
+; CHECK-NEXT: subs x9, x1, x0
+; CHECK-NEXT: mov w8, #65535 // =0xffff
+; CHECK-NEXT: csel x9, xzr, x9, lo
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: csel x8, x9, x8, lo
+; CHECK-NEXT: dup v0.4h, w8
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: mov d1, v0.d[1]
; CHECK-NEXT: // fake_use: $d0
+; CHECK-NEXT: // fake_use: $d1
; CHECK-NEXT: ret
%r = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 %i, i64 %n)
%v0 = call <2 x i1> @llvm.vector.extract.v2i1.v4i1.i64(<4 x i1> %r, i64 0)
diff --git a/llvm/test/CodeGen/AArch64/neon-get-active-lane-mask.ll b/llvm/test/CodeGen/AArch64/neon-get-active-lane-mask.ll
new file mode 100644
index 0000000000000..36d8aaeb383f9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-get-active-lane-mask.ll
@@ -0,0 +1,400 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v32i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: adrp x10, .LCPI0_5
+; CHECK-NEXT: dup v1.4s, w0
+; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT: adrp x8, .LCPI0_1
+; CHECK-NEXT: ldr q5, [x10, :lo12:.LCPI0_5]
+; CHECK-NEXT: ldr q3, [x8, :lo12:.LCPI0_1]
+; CHECK-NEXT: adrp x8, .LCPI0_4
+; CHECK-NEXT: adrp x10, .LCPI0_7
+; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI0_4]
+; CHECK-NEXT: adrp x8, .LCPI0_6
+; CHECK-NEXT: ldr q7, [x10, :lo12:.LCPI0_7]
+; CHECK-NEXT: ldr q6, [x8, :lo12:.LCPI0_6]
+; CHECK-NEXT: adrp x9, .LCPI0_2
+; CHECK-NEXT: adrp x8, .LCPI0_3
+; CHECK-NEXT: uqadd v4.4s, v1.4s, v4.4s
+; CHECK-NEXT: uqadd v5.4s, v1.4s, v5.4s
+; CHECK-NEXT: uqadd v7.4s, v1.4s, v7.4s
+; CHECK-NEXT: uqadd v6.4s, v1.4s, v6.4s
+; CHECK-NEXT: dup v2.4s, w1
+; CHECK-NEXT: ldr q16, [x9, :lo12:.LCPI0_2]
+; CHECK-NEXT: ldr q17, [x8, :lo12:.LCPI0_3]
+; CHECK-NEXT: uqadd v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: uqadd v3.4s, v1.4s, v3.4s
+; CHECK-NEXT: uqadd v16.4s, v1.4s, v16.4s
+; CHECK-NEXT: uqadd v1.4s, v1.4s, v17.4s
+; CHECK-NEXT: cmhi v4.4s, v2.4s, v4.4s
+; CHECK-NEXT: cmhi v5.4s, v2.4s, v5.4s
+; CHECK-NEXT: cmhi v6.4s, v2.4s, v6.4s
+; CHECK-NEXT: cmhi v7.4s, v2.4s, v7.4s
+; CHECK-NEXT: cmhi v0.4s, v2.4s, v0.4s
+; CHECK-NEXT: cmhi v3.4s, v2.4s, v3.4s
+; CHECK-NEXT: cmhi v16.4s, v2.4s, v16.4s
+; CHECK-NEXT: cmhi v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: uzp1 v2.8h, v5.8h, v4.8h
+; CHECK-NEXT: uzp1 v4.8h, v7.8h, v6.8h
+; CHECK-NEXT: uzp1 v0.8h, v3.8h, v0.8h
+; CHECK-NEXT: uzp1 v1.8h, v1.8h, v16.8h
+; CHECK-NEXT: uzp1 v2.16b, v4.16b, v2.16b
+; CHECK-NEXT: uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: // fake_use: $q2
+; CHECK-NEXT: // fake_use: $q0
+; CHECK-NEXT: ret
+ %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 %index, i32 %TC)
+ call void (...) @llvm.fake.use(<32 x i1> %active.lane.mask)
+ ret void
+}
+
+define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v16i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: subs w9, w1, w0
+; CHECK-NEXT: mov w8, #255 // =0xff
+; CHECK-NEXT: csel w9, wzr, w9, lo
+; CHECK-NEXT: cmp w9, #255
+; CHECK-NEXT: csel w8, w9, w8, lo
+; CHECK-NEXT: dup v0.16b, w8
+; CHECK-NEXT: adrp x8, .LCPI1_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %TC)
+ ret <16 x i1> %active.lane.mask
+}
+
+define <8 x i1> @lane_mask_v8i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v8i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: subs w9, w1, w0
+; CHECK-NEXT: mov w8, #255 // =0xff
+; CHECK-NEXT: csel w9, wzr, w9, lo
+; CHECK-NEXT: cmp w9, #255
+; CHECK-NEXT: csel w8, w9, w8, lo
+; CHECK-NEXT: dup v0.8b, w8
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %TC)
+ ret <8 x i1> %active.lane.mask
+}
+
+define <4 x i1> @lane_mask_v4i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v4i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: dup v0.4s, w0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: uqadd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: dup v1.4s, w1
+; CHECK-NEXT: cmhi v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+ %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %TC)
+ ret <4 x i1> %active.lane.mask
+}
+
+define <2 x i1> @lane_mask_v2i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v2i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov x8, #4294967296 // =0x100000000
+; CHECK-NEXT: dup v0.2s, w0
+; CHECK-NEXT: fmov d1, x8
+; CHECK-NEXT: uqadd v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: dup v1.2s, w1
+; CHECK-NEXT: cmhi v0.2s, v1.2s, v0.2s
+; CHECK-NEXT: ret
+ %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 %index, i32 %TC)
+ ret <2 x i1> %active.lane.mask
+}
+
+define <1 x i1> @lane_mask_v1i1_i32(i32 %index, i32 %TC) {
+; CHECK-LABEL: lane_mask_v1i1_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp w0, w1
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %active.lane.mask = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i32(i32 %index, i32 %TC)
+ ret <1 x i1> %active.lane.mask
+}
+
+define void @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v32i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI6_0
+; CHECK-NEXT: adrp x9, .LCPI6_2
+; CHECK-NEXT: dup v1.2d, x0
+; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT: adrp x8, .LCPI6_1
+; CHECK-NEXT: ldr q4, [x9, :lo12:.LCPI6_2]
+; CHECK-NEXT: ldr q3, [x8, :lo12:.LCPI6_1]
+; CHECK-NEXT: adrp x8, .LCPI6_3
+; CHECK-NEXT: adrp x9, .LCPI6_6
+; CHECK-NEXT: ldr q5, [x8, :lo12:.LCPI6_3]
+; CHECK-NEXT: adrp x8, .LCPI6_4
+; CHECK-NEXT: ldr q16, [x9, :lo12:.LCPI6_6]
+; CHECK-NEXT: ldr q6, [x8, :lo12:.LCPI6_4]
+; CHECK-NEXT: adrp x8, .LCPI6_5
+; CHECK-NEXT: adrp x9, .LCPI6_9
+; CHECK-NEXT: ldr q7, [x8, :lo12:.LCPI6_5]
+; CHECK-NEXT: adrp x8, .LCPI6_7
+; CHECK-NEXT: ldr q19, [x9, :lo12:.LCPI6_9]
+; CHECK-NEXT: ldr q17, [x8, :lo12:.LCPI6_7]
+; CHECK-NEXT: adrp x8, .LCPI6_8
+; CHECK-NEXT: adrp x9, .LCPI6_12
+; CHECK-NEXT: ldr q18, [x8, :lo12:.LCPI6_8]
+; CHECK-NEXT: adrp x8, .LCPI6_10
+; CHECK-NEXT: ldr q22, [x9, :lo12:.LCPI6_12]
+; CHECK-NEXT: ldr q20, [x8, :lo12:.LCPI6_10]
+; CHECK-NEXT: adrp x8, .LCPI6_11
+; CHECK-NEXT: adrp x9, .LCPI6_15
+; CHECK-NEXT: ldr q21, [x8, :lo12:.LCPI6_11]
+; CHECK-NEXT: adrp x8, .LCPI6_13
+; CHECK-NEXT: ldr q25, [x9, :lo12:.LCPI6_15]
+; CHECK-NEXT: ldr q23, [x8, :lo12:.LCPI6_13]
+; CHECK-NEXT: adrp x8, .LCPI6_14
+; CHECK-NEXT: uqadd v2.2d, v1.2d, v2.2d
+; CHECK-NEXT: ldr q24, [x8, :lo12:.LCPI6_14]
+; CHECK-NEXT: uqadd v3.2d, v1.2d, v3.2d
+; CHECK-NEXT: uqadd v4.2d, v1.2d, v4.2d
+; CHECK-NEXT: uqadd v5.2d, v1.2d, v5.2d
+; CHECK-NEXT: uqadd v6.2d, v1.2d, v6.2d
+; CHECK-NEXT: uqadd v7.2d, v1.2d, v7.2d
+; CHECK-NEXT: uqadd v16.2d, v1.2d, v16.2d
+; CHECK-NEXT: uqadd v17.2d, v1.2d, v17.2d
+; CHECK-NEXT: uqadd v18.2d, v1.2d, v18.2d
+; CHECK-NEXT: uqadd v19.2d, v1.2d, v19.2d
+; CHECK-NEXT: uqadd v20.2d, v1.2d, v20.2d
+; CHECK-NEXT: uqadd v21.2d, v1.2d, v21.2d
+; CHECK-NEXT: uqadd v22.2d, v1.2d, v22.2d
+; CHECK-NEXT: uqadd v23.2d, v1.2d, v23.2d
+; CHECK-NEXT: uqadd v24.2d, v1.2d, v24.2d
+; CHECK-NEXT: uqadd v1.2d, v1.2d, v25.2d
+; CHECK-NEXT: dup v0.2d, x1
+; CHECK-NEXT: cmhi v2.2d, v0.2d, v2.2d
+; CHECK-NEXT: cmhi v3.2d, v0.2d, v3.2d
+; CHECK-NEXT: cmhi v18.2d, v0.2d, v18.2d
+; CHECK-NEXT: cmhi v19.2d, v0.2d, v19.2d
+; CHECK-NEXT: cmhi v20.2d, v0.2d, v20.2d
+; CHECK-NEXT: cmhi v21.2d, v0.2d, v21.2d
+; CHECK-NEXT: cmhi v22.2d, v0.2d, v22.2d
+; CHECK-NEXT: cmhi v23.2d, v0.2d, v23.2d
+; CHECK-NEXT: cmhi v24.2d, v0.2d, v24.2d
+; CHECK-NEXT: cmhi v1.2d, v0.2d, v1.2d
+; CHECK-NEXT: cmhi v4.2d, v0.2d, v4.2d
+; CHECK-NEXT: cmhi v5.2d, v0.2d, v5.2d
+; CHECK-NEXT: cmhi v6.2d, v0.2d, v6.2d
+; CHECK-NEXT: cmhi v7.2d, v0.2d, v7.2d
+; CHECK-NEXT: cmhi v16.2d, v0.2d, v16.2d
+; CHECK-NEXT: cmhi v0.2d, v0.2d, v17.2d
+; CHECK-NEXT: uzp1 v2.4s, v3.4s, v2.4s
+; CHECK-NEXT: uzp1 v3.4s, v19.4s, v18.4s
+; CHECK-NEXT: uzp1 v17.4s, v21.4s, v20.4s
+; CHECK-NEXT: uzp1 v18.4s, v23.4s, v22.4s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v24.4s
+; CHECK-NEXT: uzp1 v4.4s, v5.4s, v4.4s
+; CHECK-NEXT: uzp1 v5.4s, v7.4s, v6.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v16.4s
+; CHECK-NEXT: uzp1 v3.8h, v17.8h, v3.8h
+; CHECK-NEXT: uzp1 v1.8h, v1.8h, v18.8h
+; CHECK-NEXT: uzp1 v2.8h, v4.8h, v2.8h
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-NEXT: uzp1 v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: // fake_use: $q1
+; CHECK-NEXT: // fake_use: $q0
+; CHECK-NEXT: ret
+ %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 %index, i64 %TC)
+ call void (...) @llvm.fake.use(<32 x i1> %active.lane.mask)
+ ret void
+}
+
+define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v16i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: subs x9, x1, x0
+; CHECK-NEXT: mov w8, #255 // =0xff
+; CHECK-NEXT: csel x9, xzr, x9, lo
+; CHECK-NEXT: cmp x9, #255
+; CHECK-NEXT: csel x8, x9, x8, lo
+; CHECK-NEXT: dup v0.16b, w8
+; CHECK-NEXT: adrp x8, .LCPI7_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 %index, i64 %TC)
+ ret <16 x i1> %active.lane.mask
+}
+
+define <8 x i1> @lane_mask_v8i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v8i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: subs x9, x1, x0
+; CHECK-NEXT: mov w8, #255 // =0xff
+; CHECK-NEXT: csel x9, xzr, x9, lo
+; CHECK-NEXT: cmp x9, #255
+; CHECK-NEXT: csel x8, x9, x8, lo
+; CHECK-NEXT: dup v0.8b, w8
+; CHECK-NEXT: adrp x8, .LCPI8_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI8_0]
+; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 %index, i64 %TC)
+ ret <8 x i1> %active.lane.mask
+}
+
+define <4 x i1> @lane_mask_v4i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v4i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: subs x9, x1, x0
+; CHECK-NEXT: mov w8, #65535 // =0xffff
+; CHECK-NEXT: csel x9, xzr, x9, lo
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: csel x8, x9, x8, lo
+; CHECK-NEXT: dup v0.4h, w8
+; CHECK-NEXT: adrp x8, .LCPI9_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI9_0]
+; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
+ %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 %index, i64 %TC)
+ ret <4 x i1> %active.lane.mask
+}
+
+define <2 x i1> @lane_mask_v2i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v2i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI10_0
+; CHECK-NEXT: dup v0.2d, x0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
+; CHECK-NEXT: uqadd v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: dup v1.2d, x1
+; CHECK-NEXT: cmhi v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: ret
+ %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 %index, i64 %TC)
+ ret <2 x i1> %active.lane.mask
+}
+
+define <1 x i1> @lane_mask_v1i1_i64(i64 %index, i64 %TC) {
+; CHECK-LABEL: lane_mask_v1i1_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x1
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %active.lane.mask = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i64(i64 %index, i64 %TC)
+ ret <1 x i1> %active.lane.mask
+}
+
+define void @lane_mask_v32i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v32i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI12_1
+; CHECK-NEXT: dup v0.16b, w0
+; CHECK-NEXT: adrp x9, .LCPI12_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI12_1]
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI12_0]
+; CHECK-NEXT: uqadd v1.16b, v0.16b, v1.16b
+; CHECK-NEXT: uqadd v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: dup v2.16b, w1
+; CHECK-NEXT: cmhi v1.16b, v2.16b, v1.16b
+; CHECK-NEXT: cmhi v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: // fake_use: $q1
+; CHECK-NEXT: // fake_use: $q0
+; CHECK-NEXT: ret
+ %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i8(i8 %index, i8 %TC)
+ call void (...) @llvm.fake.use(<32 x i1> %active.lane.mask)
+ ret void
+}
+
+define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v16i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI13_0
+; CHECK-NEXT: dup v0.16b, w0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI13_0]
+; CHECK-NEXT: uqadd v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: dup v1.16b, w1
+; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i8(i8 %index, i8 %TC)
+ ret <16 x i1> %active.lane.mask
+}
+
+define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v8i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dup v0.8b, w0
+; CHECK-NEXT: adrp x8, .LCPI14_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI14_0]
+; CHECK-NEXT: uqadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: dup v1.8b, w1
+; CHECK-NEXT: cmhi v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: ret
+ %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i8(i8 %index, i8 %TC)
+ ret <8 x i1> %active.lane.mask
+}
+
+define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v4i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dup v0.4h, w0
+; CHECK-NEXT: adrp x8, .LCPI15_0
+; CHECK-NEXT: movi d2, #0xff00ff00ff00ff
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI15_0]
+; CHECK-NEXT: dup v3.4h, w1
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NEXT: add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: umin v0.4h, v0.4h, v2.4h
+; CHECK-NEXT: cmhi v0.4h, v3.4h, v0.4h
+; CHECK-NEXT: ret
+ %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC)
+ ret <4 x i1> %active.lane.mask
+}
+
+define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v2i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d0, #0x0000ff000000ff
+; CHECK-NEXT: dup v1.2s, w0
+; CHECK-NEXT: mov x8, #4294967296 // =0x100000000
+; CHECK-NEXT: fmov d3, x8
+; CHECK-NEXT: dup v2.2s, w1
+; CHECK-NEXT: and v1.8b, v1.8b, v0.8b
+; CHECK-NEXT: and v2.8b, v2.8b, v0.8b
+; CHECK-NEXT: add v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: umin v0.2s, v1.2s, v0.2s
+; CHECK-NEXT: cmhi v0.2s, v2.2s, v0.2s
+; CHECK-NEXT: ret
+ %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC)
+ ret <2 x i1> %active.lane.mask
+}
+
+define <1 x i1> @lane_mask_v1i1_i8(i8 %index, i8 %TC) {
+; CHECK-LABEL: lane_mask_v1i1_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: and w8, w0, #0xff
+; CHECK-NEXT: cmp w8, w1, uxtb
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %active.lane.mask = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i8(i8 %index, i8 %TC)
+ ret <1 x i1> %active.lane.mask
+}
+
+define <8 x i1> @lane_mask_v8i1_imm3() {
+; CHECK-LABEL: lane_mask_v8i1_imm3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d0, #0x00000000ffffff
+; CHECK-NEXT: ret
+ %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 3)
+ ret <8 x i1> %active.lane.mask
+}
More information about the llvm-commits
mailing list