[llvm] ad86f7d - [AArch64] Use brk{a,b} for a lane mask from cttz.elts (#178674)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 5 02:19:28 PST 2026
Author: Graham Hunter
Date: 2026-02-05T10:19:22Z
New Revision: ad86f7dda3e0bda4f58650c2df1111aef0c57fcf
URL: https://github.com/llvm/llvm-project/commit/ad86f7dda3e0bda4f58650c2df1111aef0c57fcf
DIFF: https://github.com/llvm/llvm-project/commit/ad86f7dda3e0bda4f58650c2df1111aef0c57fcf.diff
LOG: [AArch64] Use brk{a,b} for a lane mask from cttz.elts (#178674)
cttz.elts is usually lowered (for SVE) to a brkb followed by a cntp. If
we then want a mask based on that (say, for early exit masking) then we
would use a whilelo from 0 to the result of cntp. But that just gives us
the same mask as the initial brkb, so we can just remove the cntp and
the whilelo.
Brka matches the extra +1 in the pattern.
Added:
llvm/test/CodeGen/AArch64/sve-mask-partition.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ea29f89d13f17..88836d6e167b8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6027,6 +6027,44 @@ static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+// Match get.active.lane.mask(0, cttz.elts(x)) -> brkb(x)
+// Match get.active.lane.mask(0, add(cttz.elts(x), 1)) -> brka(x)
+static SDValue optimizeBrk(SDNode *N, SelectionDAG &DAG) {
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ // Lower bound must be 0.
+ if (!isZeroOrZeroSplat(N->getOperand(0)))
+ return SDValue();
+
+ SDValue Upper = N->getOperand(1);
+
+ // Default to brkb, switch to brka if we find a +1.
+ unsigned BrkID = Intrinsic::aarch64_sve_brkb_z;
+ if (Upper->getOpcode() == ISD::ADD && isOneOrOneSplat(Upper.getOperand(1))) {
+ Upper = Upper.getOperand(0);
+ BrkID = Intrinsic::aarch64_sve_brka_z;
+ }
+
+ // We're looking for an upper bound based on CTTZ_ELTS; this would be selected
+ // as a cntp(brk(Pg, Mask)), but if we're just going to make a whilelo based
+ // on that then we just need the brk.
+ if (Upper.getOpcode() != AArch64ISD::CTTZ_ELTS || !VT.isScalableVector())
+ return SDValue();
+
+ SDValue Mask = Upper->getOperand(0);
+ const APInt &PgPattern = Upper.getConstantOperandAPInt(1);
+ SDValue Pg = getPTrue(DAG, DL, VT, PgPattern.getZExtValue());
+
+ // brk{a,b} only support .b forms, so cast to make sure all our p regs match.
+ Pg = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Pg);
+ SDValue MaskR =
+ DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Mask);
+ SDValue ID = DAG.getTargetConstant(BrkID, DL, MVT::i64);
+ SDValue Brk =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv16i1, ID, Pg, MaskR);
+ return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Brk);
+}
+
// Returns a safe bitcast between two scalable vector predicates, where
// any newly created lanes from a widening bitcast are defined as zero.
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
@@ -19471,6 +19509,9 @@ performActiveLaneMaskCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
if (DCI.isBeforeLegalize())
return SDValue();
+ if (SDValue Brk = optimizeBrk(N, DCI.DAG))
+ return Brk;
+
if (SDValue While = optimizeIncrementingWhile(N, DCI.DAG, /*IsSigned=*/false,
/*IsEqual=*/false))
return While;
diff --git a/llvm/test/CodeGen/AArch64/sve-mask-partition.ll b/llvm/test/CodeGen/AArch64/sve-mask-partition.ll
new file mode 100644
index 0000000000000..8b712bd7e42a7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-mask-partition.ll
@@ -0,0 +1,560 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
+
+;; Scalable
+define <vscale x 16 x i1> @mask_exclude_active_nxv16(<vscale x 16 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %tz.elts)
+ ret <vscale x 16 x i1> %mask.out
+}
+
+define <vscale x 8 x i1> @mask_exclude_active_nxv8(<vscale x 8 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %tz.elts)
+ ret <vscale x 8 x i1> %mask.out
+}
+
+define <vscale x 4 x i1> @mask_exclude_active_nxv4(<vscale x 4 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 %tz.elts)
+ ret <vscale x 4 x i1> %mask.out
+}
+
+define <vscale x 2 x i1> @mask_exclude_active_nxv2(<vscale x 2 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 %tz.elts)
+ ret <vscale x 2 x i1> %mask.out
+}
+
+define <vscale x 16 x i1> @mask_include_active_nxv16(<vscale x 16 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_nxv16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: brka p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %inc)
+ ret <vscale x 16 x i1> %mask.out
+}
+
+define <vscale x 8 x i1> @mask_include_active_nxv8(<vscale x 8 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_nxv8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: brka p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %inc)
+ ret <vscale x 8 x i1> %mask.out
+}
+
+define <vscale x 4 x i1> @mask_include_active_nxv4(<vscale x 4 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_nxv4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: brka p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 %inc)
+ ret <vscale x 4 x i1> %mask.out
+}
+
+define <vscale x 2 x i1> @mask_include_active_nxv2(<vscale x 2 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_nxv2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: brka p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 %inc)
+ ret <vscale x 2 x i1> %mask.out
+}
+
+;; Fixed
+define <16 x i1> @mask_exclude_active_v16(<16 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_v16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: brkb p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> %mask.in, i1 false)
+ %mask.out = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %tz.elts)
+ ret <16 x i1> %mask.out
+}
+
+define <8 x i1> @mask_exclude_active_v8(<8 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_v8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: brkb p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> %mask.in, i1 false)
+ %mask.out = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 %tz.elts)
+ ret <8 x i1> %mask.out
+}
+
+define <4 x i1> @mask_exclude_active_v4(<4 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_v4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: brkb p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> %mask.in, i1 false)
+ %mask.out = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %tz.elts)
+ ret <4 x i1> %mask.out
+}
+
+define <2 x i1> @mask_exclude_active_v2(<2 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_v2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #31
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: brkb p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> %mask.in, i1 false)
+ %mask.out = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %tz.elts)
+ ret <2 x i1> %mask.out
+}
+
+define <16 x i1> @mask_include_active_v16(<16 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_v16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: brka p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %inc)
+ ret <16 x i1> %mask.out
+}
+
+define <8 x i1> @mask_include_active_v8(<8 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_v8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: brka p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 %inc)
+ ret <8 x i1> %mask.out
+}
+
+define <4 x i1> @mask_include_active_v4(<4 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_v4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: brka p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %inc)
+ ret <4 x i1> %mask.out
+}
+
+define <2 x i1> @mask_include_active_v2(<2 x i1> %mask.in) {
+; CHECK-LABEL: mask_include_active_v2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #31
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: brka p0.b, p0/z, p1.b
+; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> %mask.in, i1 false)
+ %inc = add i64 %tz.elts, 1
+ %mask.out = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %inc)
+ ret <2 x i1> %mask.out
+}
+
+;; Wider-than-legal tests
+define <vscale x 32 x i1> @mask_exclude_active_nxv32(<vscale x 32 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-9
+; CHECK-NEXT: str p11, [sp] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #1, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #2, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z15, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z14, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z13, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z11, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x40, 0x1c // $d12 @ cfa - 40 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x40, 0x1c // $d13 @ cfa - 48 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
+; CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
+; CHECK-NEXT: index z2.d, #0, #-1
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: punpkhi p5.h, p0.b
+; CHECK-NEXT: neg x8, x8
+; CHECK-NEXT: punpkhi p4.h, p1.b
+; CHECK-NEXT: cntw x9
+; CHECK-NEXT: mov z0.d, x8
+; CHECK-NEXT: punpklo p3.h, p5.b
+; CHECK-NEXT: rdvl x8, #-1
+; CHECK-NEXT: punpklo p2.h, p4.b
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: neg x8, x9
+; CHECK-NEXT: incd z2.d, all, mul #16
+; CHECK-NEXT: punpklo p10.h, p0.b
+; CHECK-NEXT: mov z5.d, x8
+; CHECK-NEXT: punpklo p9.h, p3.b
+; CHECK-NEXT: cntd x8
+; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: neg x8, x8
+; CHECK-NEXT: add z4.d, z2.d, z0.d
+; CHECK-NEXT: punpklo p8.h, p2.b
+; CHECK-NEXT: mov z7.d, p9/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: punpklo p6.h, p10.b
+; CHECK-NEXT: mov z28.d, x8
+; CHECK-NEXT: add z25.d, z2.d, z5.d
+; CHECK-NEXT: punpklo p7.h, p1.b
+; CHECK-NEXT: mov z3.d, p8/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z6.d, z4.d, z1.d
+; CHECK-NEXT: punpklo p8.h, p6.b
+; CHECK-NEXT: and z4.d, z4.d, z7.d
+; CHECK-NEXT: punpkhi p0.h, p1.b
+; CHECK-NEXT: add z28.d, z2.d, z28.d
+; CHECK-NEXT: add z26.d, z25.d, z0.d
+; CHECK-NEXT: punpkhi p1.h, p10.b
+; CHECK-NEXT: ldr p10, [sp, #1, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z7.d, p8/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: punpklo p11.h, p7.b
+; CHECK-NEXT: and z3.d, z6.d, z3.d
+; CHECK-NEXT: add z6.d, z2.d, z1.d
+; CHECK-NEXT: punpklo p9.h, p0.b
+; CHECK-NEXT: add z29.d, z25.d, z1.d
+; CHECK-NEXT: add z5.d, z28.d, z5.d
+; CHECK-NEXT: punpklo p8.h, p1.b
+; CHECK-NEXT: mov z24.d, p11/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p11, [sp] // 2-byte Reload
+; CHECK-NEXT: punpkhi p5.h, p5.b
+; CHECK-NEXT: mov z27.d, p9/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z31.d, z26.d, z1.d
+; CHECK-NEXT: punpkhi p4.h, p4.b
+; CHECK-NEXT: mov z30.d, p8/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z2.d, z2.d, z7.d
+; CHECK-NEXT: punpklo p9.h, p5.b
+; CHECK-NEXT: and z6.d, z6.d, z24.d
+; CHECK-NEXT: add z12.d, z5.d, z1.d
+; CHECK-NEXT: punpklo p8.h, p4.b
+; CHECK-NEXT: and z7.d, z29.d, z27.d
+; CHECK-NEXT: add z29.d, z28.d, z0.d
+; CHECK-NEXT: mov z24.d, p9/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p9, [sp, #2, mul vl] // 2-byte Reload
+; CHECK-NEXT: punpkhi p3.h, p3.b
+; CHECK-NEXT: mov z8.d, p8/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Reload
+; CHECK-NEXT: punpkhi p2.h, p2.b
+; CHECK-NEXT: add z0.d, z5.d, z0.d
+; CHECK-NEXT: punpkhi p7.h, p7.b
+; CHECK-NEXT: and z25.d, z25.d, z30.d
+; CHECK-NEXT: punpkhi p6.h, p6.b
+; CHECK-NEXT: and z24.d, z26.d, z24.d
+; CHECK-NEXT: mov z10.d, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z26.d, z31.d, z8.d
+; CHECK-NEXT: punpkhi p1.h, p1.b
+; CHECK-NEXT: mov z8.d, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: add z27.d, z28.d, z1.d
+; CHECK-NEXT: mov z30.d, p7/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: punpkhi p3.h, p5.b
+; CHECK-NEXT: mov z31.d, p6/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: punpkhi p2.h, p4.b
+; CHECK-NEXT: add z9.d, z29.d, z1.d
+; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z11.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: mov z13.d, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z14.d, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z1.d, z0.d, z1.d
+; CHECK-NEXT: mov z15.d, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z27.d, z27.d, z30.d
+; CHECK-NEXT: and z28.d, z28.d, z31.d
+; CHECK-NEXT: and z29.d, z29.d, z8.d
+; CHECK-NEXT: ldr z8, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: and z30.d, z9.d, z10.d
+; CHECK-NEXT: ldr z10, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: and z5.d, z5.d, z11.d
+; CHECK-NEXT: ldr z11, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: and z31.d, z12.d, z13.d
+; CHECK-NEXT: ldr z13, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: and z0.d, z0.d, z14.d
+; CHECK-NEXT: ldr z14, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: and z1.d, z1.d, z15.d
+; CHECK-NEXT: ldr z15, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: umax z3.d, p0/m, z3.d, z4.d
+; CHECK-NEXT: umax z2.d, p0/m, z2.d, z6.d
+; CHECK-NEXT: umax z7.d, p0/m, z7.d, z25.d
+; CHECK-NEXT: umax z24.d, p0/m, z24.d, z26.d
+; CHECK-NEXT: umax z27.d, p0/m, z27.d, z28.d
+; CHECK-NEXT: umax z29.d, p0/m, z29.d, z30.d
+; CHECK-NEXT: umax z5.d, p0/m, z5.d, z31.d
+; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: umax z2.d, p0/m, z2.d, z3.d
+; CHECK-NEXT: umax z7.d, p0/m, z7.d, z24.d
+; CHECK-NEXT: umax z27.d, p0/m, z27.d, z29.d
+; CHECK-NEXT: umax z0.d, p0/m, z0.d, z5.d
+; CHECK-NEXT: umax z2.d, p0/m, z2.d, z7.d
+; CHECK-NEXT: umax z0.d, p0/m, z0.d, z27.d
+; CHECK-NEXT: umax z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: umaxv d0, p0, z0.d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: sub x8, x9, x8
+; CHECK-NEXT: rdvl x9, #1
+; CHECK-NEXT: whilelo p0.b, xzr, x8
+; CHECK-NEXT: whilelo p1.b, x9, x8
+; CHECK-NEXT: addvl sp, sp, #9
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 %tz.elts)
+ ret <vscale x 32 x i1> %mask.out
+}
+
+define <32 x i1> @mask_exclude_active_v32(<32 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_v32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr w9, [sp, #64]
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: ldr w10, [sp, #72]
+; CHECK-NEXT: index z2.b, #0, #-1
+; CHECK-NEXT: fmov s1, w9
+; CHECK-NEXT: ldr w9, [sp, #80]
+; CHECK-NEXT: mov v0.b[1], w1
+; CHECK-NEXT: mov v1.b[1], w10
+; CHECK-NEXT: ldr w10, [sp, #128]
+; CHECK-NEXT: mov z3.d, z2.d
+; CHECK-NEXT: add z2.b, z2.b, #32 // =0x20
+; CHECK-NEXT: mov v0.b[2], w2
+; CHECK-NEXT: add z3.b, z3.b, #16 // =0x10
+; CHECK-NEXT: mov v1.b[2], w9
+; CHECK-NEXT: ldr w9, [sp, #88]
+; CHECK-NEXT: mov v0.b[3], w3
+; CHECK-NEXT: mov v1.b[3], w9
+; CHECK-NEXT: ldr w9, [sp, #96]
+; CHECK-NEXT: mov v0.b[4], w4
+; CHECK-NEXT: mov v1.b[4], w9
+; CHECK-NEXT: ldr w9, [sp, #104]
+; CHECK-NEXT: mov v0.b[5], w5
+; CHECK-NEXT: mov v1.b[5], w9
+; CHECK-NEXT: ldr w9, [sp, #112]
+; CHECK-NEXT: mov v0.b[6], w6
+; CHECK-NEXT: mov v1.b[6], w9
+; CHECK-NEXT: ldr w9, [sp, #120]
+; CHECK-NEXT: mov v0.b[7], w7
+; CHECK-NEXT: mov v1.b[7], w9
+; CHECK-NEXT: ldr w9, [sp]
+; CHECK-NEXT: mov v0.b[8], w9
+; CHECK-NEXT: ldr w9, [sp, #8]
+; CHECK-NEXT: mov v1.b[8], w10
+; CHECK-NEXT: ldr w10, [sp, #136]
+; CHECK-NEXT: mov v0.b[9], w9
+; CHECK-NEXT: ldr w9, [sp, #16]
+; CHECK-NEXT: mov v1.b[9], w10
+; CHECK-NEXT: ldr w10, [sp, #144]
+; CHECK-NEXT: mov v0.b[10], w9
+; CHECK-NEXT: ldr w9, [sp, #24]
+; CHECK-NEXT: mov v1.b[10], w10
+; CHECK-NEXT: ldr w10, [sp, #152]
+; CHECK-NEXT: mov v0.b[11], w9
+; CHECK-NEXT: ldr w9, [sp, #32]
+; CHECK-NEXT: mov v1.b[11], w10
+; CHECK-NEXT: ldr w10, [sp, #160]
+; CHECK-NEXT: mov v0.b[12], w9
+; CHECK-NEXT: ldr w9, [sp, #40]
+; CHECK-NEXT: mov v1.b[12], w10
+; CHECK-NEXT: ldr w10, [sp, #168]
+; CHECK-NEXT: mov v0.b[13], w9
+; CHECK-NEXT: ldr w9, [sp, #48]
+; CHECK-NEXT: mov v1.b[13], w10
+; CHECK-NEXT: ldr w10, [sp, #176]
+; CHECK-NEXT: mov v0.b[14], w9
+; CHECK-NEXT: ldr w9, [sp, #56]
+; CHECK-NEXT: mov v1.b[14], w10
+; CHECK-NEXT: ldr w10, [sp, #184]
+; CHECK-NEXT: mov v0.b[15], w9
+; CHECK-NEXT: mov w9, #32 // =0x20
+; CHECK-NEXT: mov v1.b[15], w10
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT: and v2.16b, v0.16b, v2.16b
+; CHECK-NEXT: index z0.d, #0, #1
+; CHECK-NEXT: and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: mov z6.d, z0.d
+; CHECK-NEXT: mov z4.d, z0.d
+; CHECK-NEXT: umax v2.16b, v2.16b, v1.16b
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z5.d, z0.d
+; CHECK-NEXT: mov z7.d, z0.d
+; CHECK-NEXT: mov z17.d, z0.d
+; CHECK-NEXT: mov z18.d, z0.d
+; CHECK-NEXT: mov z19.d, z0.d
+; CHECK-NEXT: mov z20.d, z0.d
+; CHECK-NEXT: mov z21.d, z0.d
+; CHECK-NEXT: umaxv b16, v2.16b
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z22.d, z0.d
+; CHECK-NEXT: mov z23.d, z0.d
+; CHECK-NEXT: add z1.d, z1.d, #14 // =0xe
+; CHECK-NEXT: add z3.d, z3.d, #12 // =0xc
+; CHECK-NEXT: add z6.d, z6.d, #10 // =0xa
+; CHECK-NEXT: add z4.d, z4.d, #8 // =0x8
+; CHECK-NEXT: add z5.d, z5.d, #4 // =0x4
+; CHECK-NEXT: add z2.d, z2.d, #6 // =0x6
+; CHECK-NEXT: add z7.d, z7.d, #2 // =0x2
+; CHECK-NEXT: add z17.d, z17.d, #30 // =0x1e
+; CHECK-NEXT: fmov w10, s16
+; CHECK-NEXT: add z18.d, z18.d, #28 // =0x1c
+; CHECK-NEXT: add z19.d, z19.d, #26 // =0x1a
+; CHECK-NEXT: add z20.d, z20.d, #24 // =0x18
+; CHECK-NEXT: add z21.d, z21.d, #22 // =0x16
+; CHECK-NEXT: add z22.d, z22.d, #20 // =0x14
+; CHECK-NEXT: add z23.d, z23.d, #18 // =0x12
+; CHECK-NEXT: sub w9, w9, w10
+; CHECK-NEXT: and x9, x9, #0xff
+; CHECK-NEXT: dup v16.2d, x9
+; CHECK-NEXT: adrp x9, .LCPI17_0
+; CHECK-NEXT: cmhi v24.2d, v16.2d, v0.2d
+; CHECK-NEXT: add z0.d, z0.d, #16 // =0x10
+; CHECK-NEXT: cmhi v1.2d, v16.2d, v1.2d
+; CHECK-NEXT: cmhi v3.2d, v16.2d, v3.2d
+; CHECK-NEXT: cmhi v6.2d, v16.2d, v6.2d
+; CHECK-NEXT: cmhi v4.2d, v16.2d, v4.2d
+; CHECK-NEXT: cmhi v17.2d, v16.2d, v17.2d
+; CHECK-NEXT: cmhi v18.2d, v16.2d, v18.2d
+; CHECK-NEXT: cmhi v19.2d, v16.2d, v19.2d
+; CHECK-NEXT: cmhi v20.2d, v16.2d, v20.2d
+; CHECK-NEXT: cmhi v21.2d, v16.2d, v21.2d
+; CHECK-NEXT: cmhi v22.2d, v16.2d, v22.2d
+; CHECK-NEXT: cmhi v23.2d, v16.2d, v23.2d
+; CHECK-NEXT: cmhi v0.2d, v16.2d, v0.2d
+; CHECK-NEXT: cmhi v2.2d, v16.2d, v2.2d
+; CHECK-NEXT: cmhi v5.2d, v16.2d, v5.2d
+; CHECK-NEXT: cmhi v7.2d, v16.2d, v7.2d
+; CHECK-NEXT: uzp1 v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: uzp1 v3.4s, v18.4s, v17.4s
+; CHECK-NEXT: uzp1 v16.4s, v20.4s, v19.4s
+; CHECK-NEXT: uzp1 v17.4s, v22.4s, v21.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v23.4s
+; CHECK-NEXT: uzp1 v4.4s, v4.4s, v6.4s
+; CHECK-NEXT: uzp1 v2.4s, v5.4s, v2.4s
+; CHECK-NEXT: uzp1 v5.4s, v24.4s, v7.4s
+; CHECK-NEXT: uzp1 v3.8h, v16.8h, v3.8h
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v17.8h
+; CHECK-NEXT: uzp1 v1.8h, v4.8h, v1.8h
+; CHECK-NEXT: uzp1 v2.8h, v5.8h, v2.8h
+; CHECK-NEXT: uzp1 v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI17_0]
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT: zip1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: str h0, [x8, #2]
+; CHECK-NEXT: str h1, [x8]
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> %mask.in, i1 false)
+ %mask.out = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %tz.elts)
+ ret <32 x i1> %mask.out
+}
+
+;; Non-matches
+define <vscale x 16 x i1> @mask_exclude_active_nxv16_nonzero_lower_bound(<vscale x 16 x i1> %mask.in) {
+; CHECK-LABEL: mask_exclude_active_nxv16_nonzero_lower_bound:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: mov w9, #1 // =0x1
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: cntp x8, p0, p0.b
+; CHECK-NEXT: whilelo p0.b, x9, x8
+; CHECK-NEXT: ret
+ %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
+ %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 1, i64 %tz.elts)
+ ret <vscale x 16 x i1> %mask.out
+}
More information about the llvm-commits
mailing list