[llvm] be1d8d4 - [AArch64] Lower cttz(bitcast <Nxi1> to iN) with shrn-based compressed movemask (#199081)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 2 18:14:07 PDT 2026
Author: Adam Scott
Date: 2026-07-02T18:14:02-07:00
New Revision: be1d8d4bb908b90810025479ec1a8afae34a630a
URL: https://github.com/llvm/llvm-project/commit/be1d8d4bb908b90810025479ec1a8afae34a630a
DIFF: https://github.com/llvm/llvm-project/commit/be1d8d4bb908b90810025479ec1a8afae34a630a.diff
LOG: [AArch64] Lower cttz(bitcast <Nxi1> to iN) with shrn-based compressed movemask (#199081)
The existing lowering in vectorToScalarBitmask() creates a 1 bit per
lane movemask using a powers of 2 reduction (and+addv with a constant
pool entry).
This patch adds a DAG combine on ISD::CTTZ that recognizes cttz(bitcast
<N x i1> to iN) and produces a compressed movemask with shrn (for i8
lanes) or xtn (for wider lanes) then runs scalar cttz on a 64- or
128-bit value. Dividing by bits per lane gives the lane index.
Supports lane counts {2, 4, 8, 16, 32} (one or two NEON registers)
For the example in the issue (`<16 x i8> -> i16`):
Before:
```asm
adrp x8, .LCPI0_0
cmlt v0.16b, v0.16b, #0
ldr q1, [x8, :lo12:.LCPI0_0]
and v0.16b, v0.16b, v1.16b
ext v1.16b, v0.16b, v0.16b, #8
zip1 v0.16b, v0.16b, v1.16b
addv h0, v0.8h
fmov w8, s0
orr w8, w8, #0x10000
rbit w8, w8
clz w0, w8
ret
```
After:
```asm
cmlt v0.16b, v0.16b, #0
shrn v0.8b, v0.8h, #4
fmov x8, d0
rbit x8, x8
clz x8, x8
lsr x0, x8, #2
ret
```
Also created a new test file with 24 functions covering both endianness
- 7 basic widths (`<16 x i8>` down through `<2 x i32>`)
- 4 wider cases that span two NEON registers (`<32 x i8>` etc)
- 2 narrow cases that get sext'd up to fit one register
- 2 cases where the bitcast has two users
- 4 alternative comparison operators (`eq`, `ne`, etc)
- 1 `cttz` with `is_zero_undef=true`
- 4 negative tests that bails for unsupported shapes
Fixes #186295
Added:
llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c4c52dbe395e8..9ef66ecad411c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1234,6 +1234,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::GlobalAddress);
setTargetDAGCombine(ISD::CTLZ);
+ setTargetDAGCombine({ISD::CTTZ, ISD::CTTZ_ZERO_POISON});
setTargetDAGCombine(ISD::GET_ACTIVE_LANE_MASK);
@@ -26715,6 +26716,134 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits);
}
+// When taking the trailing-zero count of a bitcast from <N x i1> to scalar
+// iN, we can pack each lane into a small bit-slot of a 64-bit scalar
+// with shrn (for byte lanes) or xtn (for wider lanes) then run a
+// scalar cttz on the result.
+static SDValue performCTTZCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG) {
+ // Run before vectorToScalarBitmask() expands it.
+ if (!DCI.isBeforeLegalize())
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ if (!VT.isScalarInteger())
+ return SDValue();
+ SDValue BitcastNode = N->getOperand(0);
+ if (BitcastNode.getOpcode() != ISD::BITCAST)
+ return SDValue();
+ SDValue Predicate = BitcastNode.getOperand(0);
+ EVT PredVT = Predicate.getValueType();
+ if (!PredVT.isFixedLengthVector() || PredVT.getVectorElementType() != MVT::i1)
+ return SDValue();
+
+ unsigned NumLanes = PredVT.getVectorNumElements();
+ if (!isPowerOf2_32(NumLanes) || NumLanes < 2 || NumLanes > 32)
+ return SDValue();
+
+ constexpr unsigned DRegBits = 64;
+ constexpr unsigned QRegBits = 128;
+ constexpr unsigned MinLaneBits = 8;
+
+ // Pick a lane width to sext the predicate to. Prefer the original
+ // compare input width if we can find it and it fits in at most two NEON
+ // registers but otherwise pick a minimum-width type that fits.
+ EVT MaskVT = tryGetOriginalBoolVectorType(Predicate);
+ if (!MaskVT.isSimple() ||
+ MaskVT.changeVectorElementTypeToInteger().getSizeInBits() >
+ 2 * QRegBits ||
+ MaskVT.changeVectorElementTypeToInteger().getSizeInBits() < DRegBits) {
+ unsigned LaneBits = std::max(DRegBits / NumLanes, MinLaneBits);
+ MaskVT = MVT::getVectorVT(MVT::getIntegerVT(LaneBits), NumLanes);
+ }
+ MaskVT = MaskVT.changeVectorElementTypeToInteger();
+
+ unsigned MaskBits = MaskVT.getSizeInBits();
+ if (MaskBits != DRegBits && MaskBits != QRegBits && MaskBits != 2 * QRegBits)
+ return SDValue();
+
+ // Target 64 bits when lanes fit, otherwise 128.
+ unsigned TargetBits = DRegBits;
+ if (MaskBits == 2 * QRegBits && NumLanes > 16)
+ TargetBits = QRegBits;
+ unsigned CompressedBits = std::min(MaskBits, TargetBits);
+
+ SDLoc DL(N);
+ SDValue Mask = DAG.getSExtOrTrunc(Predicate, DL, MaskVT);
+
+ LLVMContext &Ctx = *DAG.getContext();
+
+ // NEON's smallest vector lane type is i8 so we can't narrow further.
+ // Instead, bitcast the mask to i16 lanes and shrn by 4 to pack each
+ // input byte into 4 bits of the output
+ auto ShrnPair = [&](SDValue V) -> SDValue {
+ EVT InVT = V.getValueType();
+ EVT OutVT = InVT.getHalfNumVectorElementsVT(Ctx);
+ EVT PairedVT = OutVT.widenIntegerVectorElementType(Ctx);
+ SDValue Paired = DAG.getNode(AArch64ISD::NVCAST, DL, PairedVT, V);
+ SDValue ShAmt = DAG.getConstant(MinLaneBits / 2, DL, PairedVT);
+ SDValue Shifted = DAG.getNode(ISD::SRL, DL, PairedVT, Paired, ShAmt);
+ return DAG.getNode(ISD::TRUNCATE, DL, OutVT, Shifted);
+ };
+
+ // Trailing-zero count of the compressed result as an i64.
+ auto CompressedCttz = [&](SDValue V) -> SDValue {
+ if (V.getValueType().getSizeInBits() == DRegBits) {
+ SDValue Nv = DAG.getNode(AArch64ISD::NVCAST, DL, MVT::v1i64, V);
+ SDValue Scalar = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Nv,
+ DAG.getConstant(0, DL, MVT::i64));
+ return DAG.getNode(ISD::CTTZ, DL, MVT::i64, Scalar);
+ }
+ SDValue Compressed = DAG.getBitcast(MVT::i128, V);
+ // BITCAST puts lane 0 at the high end of the scalar on BE (unlike NVCAST
+ // above), so use clz instead of cttz to find the first match
+ unsigned Op = DAG.getDataLayout().isLittleEndian() ? ISD::CTTZ : ISD::CTLZ;
+ SDValue Ctz = DAG.getNode(Op, DL, MVT::i128, Compressed);
+ return DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Ctz);
+ };
+
+ SDValue Cur = Mask;
+ // Narrow wide lanes toward CompressedBits with xtn.
+ while (Cur.getValueType().getScalarSizeInBits() > MinLaneBits &&
+ Cur.getValueType().getSizeInBits() > CompressedBits) {
+ EVT CurVT = Cur.getValueType();
+ unsigned CurLaneBits = CurVT.getScalarSizeInBits();
+ EVT NarrowVT = CurVT.changeVectorElementType(
+ Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2));
+ Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur);
+ }
+
+ // If still too wide, pack two lanes per byte with shrn.
+ if (Cur.getValueType().getSizeInBits() > CompressedBits) {
+ EVT CurVT = Cur.getValueType();
+ if (CurVT.getSizeInBits() <= QRegBits) {
+ Cur = ShrnPair(Cur);
+ } else {
+ // Split into Q-reg-sized halves so each ShrnPair stays in range.
+ EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx);
+ SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(0, DL, MVT::i64));
+ SDValue Hi = DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64));
+ Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo),
+ ShrnPair(Hi));
+ }
+ }
+
+ // Cur has been narrowed to CompressedBits.
+ assert(Cur.getValueType().getSizeInBits() == CompressedBits &&
+ "Unexpected mask width!");
+
+ SDValue Ctz = CompressedCttz(Cur);
+ unsigned LaneIndexShift = Log2_32(CompressedBits / NumLanes);
+ if (LaneIndexShift)
+ Ctz = DAG.getNode(ISD::SRL, DL, MVT::i64, Ctz,
+ DAG.getShiftAmountConstant(LaneIndexShift, MVT::i64, DL));
+ return DAG.getSExtOrTrunc(Ctz, DL, VT);
+}
+
static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG,
StoreSDNode *Store) {
if (!Store->isTruncatingStore())
@@ -30280,6 +30409,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performMINMAXCombine(N, DAG, *this);
case ISD::TRUNCATE:
return performTruncateCombine(N, DAG, DCI);
+ case ISD::CTTZ:
+ case ISD::CTTZ_ZERO_POISON:
+ return performCTTZCombine(N, DCI, DAG);
case AArch64ISD::ANDS:
return performANDSCombine(N, DCI);
case AArch64ISD::ADC:
diff --git a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
new file mode 100644
index 0000000000000..f7186dbd2cc7c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
@@ -0,0 +1,940 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+define i16 @cttz_v16i8(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i16(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i8 @cttz_v8i8(<8 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v8i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i8> %v, splat (i8 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i16 @cttz_v16i8_with_any_check(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %any = icmp ne i16 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+notfound:
+ ret i16 16
+}
+
+
+define i4 @cttz_v4i32(<4 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i32> %v, splat (i32 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i2 @cttz_v2i64(<2 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v2i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: xtn v0.2s, v0.2d
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: xtn v0.2s, v0.2d
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i64> %v, splat (i64 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+
+define i16 @cttz_v16i8_eq(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_eq:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #42
+; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_eq:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #42
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp eq <16 x i8> %v, splat (i8 42)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_ne(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_ne:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_ne:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp ne <16 x i8> %v, zeroinitializer
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_sgt(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_sgt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #5
+; CHECK-LE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_sgt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #5
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp sgt <16 x i8> %v, splat (i8 5)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+
+define i4 @cttz_v4f32_olt(<4 x float> %v) {
+; CHECK-LE-LABEL: cttz_v4f32_olt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4f32_olt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = fcmp olt <4 x float> %v, zeroinitializer
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+
+define i8 @cttz_v8i16_with_any_check(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %any = icmp ne i8 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 true)
+ ret i8 %ctz
+notfound:
+ ret i8 8
+}
+
+
+define i16 @cttz_v16i8_zero_undef(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+}
+
+
+define i32 @cttz_plain_i32(i32 %x) {
+; CHECK-LABEL: cttz_plain_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rbit w8, w0
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %ctz = call i32 @llvm.cttz.i32(i32 %x, i1 false)
+ ret i32 %ctz
+}
+
+define i32 @cttz_bitcast_from_float(float %f) {
+; CHECK-LABEL: cttz_bitcast_from_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: rbit w8, w8
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %i = bitcast float %f to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %i, i1 false)
+ ret i32 %ctz
+}
+
+define <4 x i32> @cttz_vector_result_no_fold(<4 x i32> %v) {
+; CHECK-LE-LABEL: cttz_vector_result_no_fold:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: rev32 v0.16b, v0.16b
+; CHECK-LE-NEXT: rbit v0.16b, v0.16b
+; CHECK-LE-NEXT: clz v0.4s, v0.4s
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_vector_result_no_fold:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: rev32 v0.16b, v0.16b
+; CHECK-BE-NEXT: rbit v0.16b, v0.16b
+; CHECK-BE-NEXT: clz v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %ctz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %v, i1 false)
+ ret <4 x i32> %ctz
+}
+
+define i3 @cttz_v3i8_non_pow2_lanes(<3 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: fmov s0, w0
+; CHECK-LE-NEXT: mov v0.h[1], w1
+; CHECK-LE-NEXT: mov v0.h[2], w2
+; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: umov w9, v0.h[1]
+; CHECK-LE-NEXT: umov w10, v0.h[2]
+; CHECK-LE-NEXT: and w8, w8, #0x1
+; CHECK-LE-NEXT: bfi w8, w9, #1, #1
+; CHECK-LE-NEXT: bfi w8, w10, #2, #1
+; CHECK-LE-NEXT: orr w8, w8, #0x8
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: fmov s0, w0
+; CHECK-BE-NEXT: mov v0.h[1], w1
+; CHECK-BE-NEXT: mov v0.h[2], w2
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: umov w8, v0.h[1]
+; CHECK-BE-NEXT: umov w9, v0.h[0]
+; CHECK-BE-NEXT: umov w10, v0.h[2]
+; CHECK-BE-NEXT: ubfiz w8, w8, #1, #1
+; CHECK-BE-NEXT: bfi w8, w9, #2, #1
+; CHECK-BE-NEXT: bfxil w8, w10, #0, #1
+; CHECK-BE-NEXT: orr w8, w8, #0x8
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <3 x i8> %v, splat (i8 0)
+ %bm = bitcast <3 x i1> %cmp to i3
+ %ctz = call i3 @llvm.cttz.i3(i3 %bm, i1 false)
+ ret i3 %ctz
+}
+declare i3 @llvm.cttz.i3(i3, i1)
+
+define i16 @cttz_v16i16(<16 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v16i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i16> %v, splat (i16 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i32(<8 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v8i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i32> %v, splat (i32 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i4 @cttz_v4i64(<4 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v4i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i64> %v, splat (i64 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i32 @cttz_v32i8(<32 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v32i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-LE-NEXT: mov x8, v0.d[1]
+; CHECK-LE-NEXT: fmov x9, d0
+; CHECK-LE-NEXT: rbit x10, x9
+; CHECK-LE-NEXT: cmp x9, #0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x10, x10
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: add x8, x8, #64
+; CHECK-LE-NEXT: csel x8, x10, x8, ne
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v32i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: mov x8, v0.d[1]
+; CHECK-BE-NEXT: fmov x9, d0
+; CHECK-BE-NEXT: clz x10, x9
+; CHECK-BE-NEXT: cmp x9, #0
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: add x8, x8, #64
+; CHECK-BE-NEXT: csel x8, x10, x8, ne
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <32 x i8> %v, splat (i8 0)
+ %bm = bitcast <32 x i1> %cmp to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %bm, i1 false)
+ ret i32 %ctz
+}
+
+define i32 @cttz_v32i16(<32 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v32i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: cmlt v3.8h, v3.8h, #0
+; CHECK-LE-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-LE-NEXT: mov x8, v0.d[1]
+; CHECK-LE-NEXT: fmov x9, d0
+; CHECK-LE-NEXT: rbit x10, x9
+; CHECK-LE-NEXT: cmp x9, #0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x10, x10
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: add x8, x8, #64
+; CHECK-LE-NEXT: csel x8, x10, x8, ne
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v32i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-NEXT: rev64 v3.8h, v3.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-BE-NEXT: cmlt v3.8h, v3.8h, #0
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: mov x8, v0.d[1]
+; CHECK-BE-NEXT: fmov x9, d0
+; CHECK-BE-NEXT: clz x10, x9
+; CHECK-BE-NEXT: cmp x9, #0
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: add x8, x8, #64
+; CHECK-BE-NEXT: csel x8, x10, x8, ne
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <32 x i16> %v, splat (i16 0)
+ %bm = bitcast <32 x i1> %cmp to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %bm, i1 false)
+ ret i32 %ctz
+}
+
+define i16 @cttz_v16i32(<16 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v16i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-LE-NEXT: cmlt v2.4s, v2.4s, #0
+; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmlt v2.4s, v2.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i32> %v, splat (i32 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i64 @cttz_v64i8_too_many_lanes(<64 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v64i8_too_many_lanes:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI22_0
+; CHECK-LE-NEXT: cmlt v3.16b, v3.16b, #0
+; CHECK-LE-NEXT: cmlt v2.16b, v2.16b, #0
+; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q4, [x8, :lo12:.LCPI22_0]
+; CHECK-LE-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-LE-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-LE-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v3.h[0]
+; CHECK-LE-NEXT: umov w9, v2.h[0]
+; CHECK-LE-NEXT: umov w10, v1.h[0]
+; CHECK-LE-NEXT: umov w11, v0.h[0]
+; CHECK-LE-NEXT: bfi w9, w8, #16, #16
+; CHECK-LE-NEXT: bfi w11, w10, #16, #16
+; CHECK-LE-NEXT: orr x8, x11, x9, lsl #32
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x0, x8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v64i8_too_many_lanes:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI22_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
+; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: cmlt v3.16b, v3.16b, #0
+; CHECK-BE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-BE-NEXT: cmlt v2.16b, v2.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-BE-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-BE-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-BE-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v3.16b, v3.16b
+; CHECK-BE-NEXT: rev16 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: umov w11, v3.h[0]
+; CHECK-BE-NEXT: umov w9, v1.h[0]
+; CHECK-BE-NEXT: umov w10, v2.h[0]
+; CHECK-BE-NEXT: bfi w9, w8, #16, #16
+; CHECK-BE-NEXT: bfi w11, w10, #16, #16
+; CHECK-BE-NEXT: orr x8, x11, x9, lsl #32
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x0, x8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <64 x i8> %v, splat (i8 0)
+ %bm = bitcast <64 x i1> %cmp to i64
+ %ctz = call i64 @llvm.cttz.i64(i64 %bm, i1 false)
+ ret i64 %ctz
+}
+
+define i4 @cttz_v4i16(<4 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v4i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i16> %v, splat (i16 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i2 @cttz_v2i32(<2 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v2i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i32> %v, splat (i32 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+define i2 @cttz_v2i8(<2 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v2i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-LE-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-BE-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i8> %v, splat (i8 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+define i4 @cttz_v4i8(<4 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v4i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i8> %v, splat (i8 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
More information about the llvm-commits
mailing list