[llvm] [AArch64] Use firstp for SVE2.2 cttz.elts (PR #221182)
Vimal Patel via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 02:17:05 PDT 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/221182
>From f1d64f942a25c7352ebf97cff4fa256c531e548a Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Thu, 3 Sep 2026 09:13:23 +0000
Subject: [PATCH] [AArch64] Use firstp for SVE2.2 cttz.elts
Preserve zero-is-poison cttz.elts as a distinct AArch64ISD node.
This lets instruction selection choose firstp for SVE2.2 and SME2.2.
Keep brkb/cntp for no-poison uses and for older targets.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 8 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 98 +++--
.../AArch64/AArch64SelectionDAGInfo.cpp | 1 +
.../AArch64/intrinsic-cttz-elts-sve.ll | 370 ++++++++++++------
5 files changed, 321 insertions(+), 159 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c51683b78cb54..6af5cc08cc47b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9153,8 +9153,12 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
}
SDValue Pg = getPredicateForVector(DAG, DL, VT);
- SDValue NewCttzElts =
- DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, Pg, CttzOp);
+ // We preserve the poison semantics here to avoid the poison path from being
+ // affected by the checks emitted for the no-poison case.
+ unsigned Opcode = Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON
+ ? AArch64ISD::CTTZ_ELTS_ZERO_POISON
+ : AArch64ISD::CTTZ_ELTS;
+ SDValue NewCttzElts = DAG.getNode(Opcode, DL, MVT::i64, Pg, CttzOp);
return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3b226e9a78105..debb0336e7c64 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1258,6 +1258,9 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
// res = cttz.elts(pg, op)
def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 2,
[SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
+def AArch64CttzEltsZeroPoison
+ : SDNode<"AArch64ISD::CTTZ_ELTS_ZERO_POISON", SDTypeProfile<1, 2,
+ [SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2987b1dc85a7c..51ed1bed6070c 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2437,21 +2437,26 @@ let Predicates = [HasSVE_or_SME] in {
defm CNTD_XPiI : sve_int_count<0b110, "cntd", int_aarch64_sve_cntd>;
defm CNTP_XPP : sve_int_pcount_pred<0b000, "cntp", int_aarch64_sve_cntp>;
- def : Pat<(i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op1)),
- (CNTP_XPP_B (BRKB_PPzP PPR:$Pg, PPR:$Op1),
- (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
-
- def : Pat<(i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op1)),
- (CNTP_XPP_H (BRKB_PPzP PPR:$Pg, PPR:$Op1),
- (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
-
- def : Pat<(i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op1)),
- (CNTP_XPP_S (BRKB_PPzP PPR:$Pg, PPR:$Op1),
- (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+ multiclass sve_cttz_elts_pats<SDPatternOperator cttz_op> {
+ def : Pat<(i64 (cttz_op nxv16i1:$Pg, nxv16i1:$Op1)),
+ (CNTP_XPP_B (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+
+ def : Pat<(i64 (cttz_op nxv8i1:$Pg, nxv8i1:$Op1)),
+ (CNTP_XPP_H (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+
+ def : Pat<(i64 (cttz_op nxv4i1:$Pg, nxv4i1:$Op1)),
+ (CNTP_XPP_S (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+
+ def : Pat<(i64 (cttz_op nxv2i1:$Pg, nxv2i1:$Op1)),
+ (CNTP_XPP_D (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+ }
- def : Pat<(i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op1)),
- (CNTP_XPP_D (BRKB_PPzP PPR:$Pg, PPR:$Op1),
- (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
+ defm : sve_cttz_elts_pats<AArch64CttzElts>;
+ defm : sve_cttz_elts_pats<AArch64CttzEltsZeroPoison>;
}
defm INCB_XPiI : sve_int_pred_pattern_a<0b000, "incb", add, int_aarch64_sve_cntb>;
@@ -2537,37 +2542,42 @@ let Predicates = [HasSVE_or_SME] in {
defm INCP_ZP : sve_int_count_v<0b10000, "incp">;
defm DECP_ZP : sve_int_count_v<0b10100, "decp">;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op2)))),
- (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
+ multiclass sve_cttz_elts_add_pats<SDPatternOperator cttz_op> {
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (cttz_op nxv16i1:$Pg, nxv16i1:$Op2)))),
+ (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op2))))),
- (EXTRACT_SUBREG (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2),
- (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
- sub_32)>;
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (cttz_op nxv16i1:$Pg, nxv16i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2),
+ (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+ sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op2)))),
- (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (cttz_op nxv8i1:$Pg, nxv8i1:$Op2)))),
+ (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op2))))),
- (EXTRACT_SUBREG (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2),
- (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
- sub_32)>;
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (cttz_op nxv8i1:$Pg, nxv8i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2),
+ (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+ sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op2)))),
- (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (cttz_op nxv4i1:$Pg, nxv4i1:$Op2)))),
+ (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op2))))),
- (EXTRACT_SUBREG (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2),
- (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
- sub_32)>;
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (cttz_op nxv4i1:$Pg, nxv4i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2),
+ (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+ sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op2)))),
- (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (cttz_op nxv2i1:$Pg, nxv2i1:$Op2)))),
+ (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op2))))),
- (EXTRACT_SUBREG (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2),
- (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
- sub_32)>;
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (cttz_op nxv2i1:$Pg, nxv2i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2),
+ (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+ sub_32)>;
+ }
+
+ defm : sve_cttz_elts_add_pats<AArch64CttzElts>;
+ defm : sve_cttz_elts_add_pats<AArch64CttzEltsZeroPoison>;
defm INDEX_RR : sve_int_index_rr<"index", AArch64mul_p_oneuse>;
defm INDEX_IR : sve_int_index_ir<"index", AArch64mul_p, AArch64mul_p_oneuse>;
@@ -5071,6 +5081,18 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
defm FIRSTP_XPP : sve_int_pcount_pred<0b001, "firstp", int_aarch64_sve_firstp>;
defm LASTP_XPP : sve_int_pcount_pred<0b010, "lastp", int_aarch64_sve_lastp>;
+ def : Pat<(i64 (AArch64CttzEltsZeroPoison nxv16i1:$Pg, nxv16i1:$Op1)),
+ (FIRSTP_XPP_B PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64CttzEltsZeroPoison nxv8i1:$Pg, nxv8i1:$Op1)),
+ (FIRSTP_XPP_H PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64CttzEltsZeroPoison nxv4i1:$Pg, nxv4i1:$Op1)),
+ (FIRSTP_XPP_S PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64CttzEltsZeroPoison nxv2i1:$Pg, nxv2i1:$Op1)),
+ (FIRSTP_XPP_D PPR:$Pg, PPR:$Op1)>;
+
// SVE reverse within elements, zeroing predicate
defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit", AArch64rbit_mt>;
defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb", AArch64revb_mt>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 7eac36af4ab46..0e9f1a00a3e36 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -49,6 +49,7 @@ void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
// Some additional checks not yet implemented by verifyTargetNode.
switch (N->getOpcode()) {
case AArch64ISD::CTTZ_ELTS:
+ case AArch64ISD::CTTZ_ELTS_ZERO_POISON:
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
"Expected the general-predicate and mask to have matching types");
break;
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
index 4a74a2587b39c..0eb1cb422d836 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
@@ -1,22 +1,25 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING
-; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING
+; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,SCALABLE1,SVE
+; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,SCALABLE1,SME
+; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,SCALABLE2p2,SVE2P2
+; RUN: llc -enable-subreg-liveness -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,SCALABLE2p2,SME2P2
; WITH VSCALE RANGE
define i32 @ctz_nxv32i1(<vscale x 32 x i1> %a) #0 {
-; NONSTREAMING-LABEL: ctz_nxv32i1:
-; NONSTREAMING: // %bb.0:
-; NONSTREAMING-NEXT: ptrue p2.b
-; NONSTREAMING-NEXT: rdvl x8, #1
-; NONSTREAMING-NEXT: mov w10, w8
-; NONSTREAMING-NEXT: brkb p0.b, p2/z, p0.b
-; NONSTREAMING-NEXT: brkb p1.b, p2/z, p1.b
-; NONSTREAMING-NEXT: cntp x9, p0, p0.b
-; NONSTREAMING-NEXT: incp x8, p1.b
-; NONSTREAMING-NEXT: cmp w9, w10
-; NONSTREAMING-NEXT: csel w0, w9, w8, ne
-; NONSTREAMING-NEXT: ret
+;
+; SVE-LABEL: ctz_nxv32i1:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p2.b
+; SVE-NEXT: rdvl x8, #1
+; SVE-NEXT: mov w10, w8
+; SVE-NEXT: brkb p0.b, p2/z, p0.b
+; SVE-NEXT: brkb p1.b, p2/z, p1.b
+; SVE-NEXT: cntp x9, p0, p0.b
+; SVE-NEXT: incp x8, p1.b
+; SVE-NEXT: cmp w9, w10
+; SVE-NEXT: csel w0, w9, w8, ne
+; SVE-NEXT: ret
;
; STREAMING-LABEL: ctz_nxv32i1:
; STREAMING: // %bb.0:
@@ -30,6 +33,19 @@ define i32 @ctz_nxv32i1(<vscale x 32 x i1> %a) #0 {
; STREAMING-NEXT: cmp w9, w10
; STREAMING-NEXT: csel w0, w9, w8, ne
; STREAMING-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv32i1:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p2.b
+; SCALABLE2p2-NEXT: rdvl x10, #1
+; SCALABLE2p2-NEXT: brkb p1.b, p2/z, p1.b
+; SCALABLE2p2-NEXT: brkb p0.b, p2/z, p0.b
+; SCALABLE2p2-NEXT: cntp x8, p1, p1.b
+; SCALABLE2p2-NEXT: cntp x9, p0, p0.b
+; SCALABLE2p2-NEXT: incb x8
+; SCALABLE2p2-NEXT: cmp w9, w10
+; SCALABLE2p2-NEXT: csel w0, w9, w8, ne
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> %a, i1 0)
ret i32 %res
}
@@ -61,13 +77,21 @@ define i64 @vscale_4096(<vscale x 16 x i8> %a) #1 {
}
define i64 @vscale_4096_poison(<vscale x 16 x i8> %a) #1 {
-; CHECK-LABEL: vscale_4096_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; CHECK-NEXT: brkb p1.b, p0/z, p1.b
-; CHECK-NEXT: cntp x0, p1, p1.b
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: vscale_4096_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p0.b
+; SCALABLE1-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SCALABLE1-NEXT: brkb p1.b, p0/z, p1.b
+; SCALABLE1-NEXT: cntp x0, p1, p1.b
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: vscale_4096_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p0.b
+; SCALABLE2p2-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SCALABLE2p2-NEXT: firstp x0, p0, p1.b
+; SCALABLE2p2-NEXT: ret
%res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i8(<vscale x 16 x i8> %a, i1 1)
ret i64 %res
}
@@ -86,12 +110,19 @@ define i32 @ctz_nxv2i1(<vscale x 2 x i1> %a) {
}
define i32 @ctz_nxv2i1_poison(<vscale x 2 x i1> %a) {
-; CHECK-LABEL: ctz_nxv2i1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p1.d
-; CHECK-NEXT: brkb p0.b, p1/z, p0.b
-; CHECK-NEXT: cntp x0, p0, p0.d
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: ctz_nxv2i1_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p1.d
+; SCALABLE1-NEXT: brkb p0.b, p1/z, p0.b
+; SCALABLE1-NEXT: cntp x0, p0, p0.d
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv2i1_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p1.d
+; SCALABLE2p2-NEXT: firstp x0, p1, p0.d
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> %a, i1 1)
ret i32 %res
}
@@ -140,12 +171,19 @@ define i32 @ctz_nxv3i1(<vscale x 3 x i1> %a) {
}
define i32 @ctz_nxv3i1_poison(<vscale x 3 x i1> %a) {
-; CHECK-LABEL: ctz_nxv3i1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: brkb p0.b, p1/z, p0.b
-; CHECK-NEXT: cntp x0, p0, p0.s
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: ctz_nxv3i1_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p1.s
+; SCALABLE1-NEXT: brkb p0.b, p1/z, p0.b
+; SCALABLE1-NEXT: cntp x0, p0, p0.s
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv3i1_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p1.s
+; SCALABLE2p2-NEXT: firstp x0, p1, p0.s
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv3i1(<vscale x 3 x i1> %a, i1 1)
ret i32 %res
}
@@ -162,12 +200,19 @@ define i32 @ctz_nxv4i1(<vscale x 4 x i1> %a) {
}
define i32 @ctz_nxv4i1_poison(<vscale x 4 x i1> %a) {
-; CHECK-LABEL: ctz_nxv4i1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: brkb p0.b, p1/z, p0.b
-; CHECK-NEXT: cntp x0, p0, p0.s
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: ctz_nxv4i1_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p1.s
+; SCALABLE1-NEXT: brkb p0.b, p1/z, p0.b
+; SCALABLE1-NEXT: cntp x0, p0, p0.s
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv4i1_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p1.s
+; SCALABLE2p2-NEXT: firstp x0, p1, p0.s
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> %a, i1 1)
ret i32 %res
}
@@ -209,12 +254,19 @@ define i32 @ctz_nxv8i1(<vscale x 8 x i1> %a) {
}
define i32 @ctz_nxv8i1_poison(<vscale x 8 x i1> %a) {
-; CHECK-LABEL: ctz_nxv8i1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p1.h
-; CHECK-NEXT: brkb p0.b, p1/z, p0.b
-; CHECK-NEXT: cntp x0, p0, p0.h
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: ctz_nxv8i1_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p1.h
+; SCALABLE1-NEXT: brkb p0.b, p1/z, p0.b
+; SCALABLE1-NEXT: cntp x0, p0, p0.h
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv8i1_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p1.h
+; SCALABLE2p2-NEXT: firstp x0, p1, p0.h
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> %a, i1 1)
ret i32 %res
}
@@ -256,12 +308,19 @@ define i32 @ctz_nxv16i1(<vscale x 16 x i1> %a) {
}
define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %a) {
-; CHECK-LABEL: ctz_nxv16i1_poison:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: brkb p0.b, p1/z, p0.b
-; CHECK-NEXT: cntp x0, p0, p0.b
-; CHECK-NEXT: ret
+;
+; SCALABLE1-LABEL: ctz_nxv16i1_poison:
+; SCALABLE1: // %bb.0:
+; SCALABLE1-NEXT: ptrue p1.b
+; SCALABLE1-NEXT: brkb p0.b, p1/z, p0.b
+; SCALABLE1-NEXT: cntp x0, p0, p0.b
+; SCALABLE1-NEXT: ret
+;
+; SCALABLE2p2-LABEL: ctz_nxv16i1_poison:
+; SCALABLE2p2: // %bb.0:
+; SCALABLE2p2-NEXT: ptrue p1.b
+; SCALABLE2p2-NEXT: firstp x0, p1, p0.b
+; SCALABLE2p2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %a, i1 1)
ret i32 %res
}
@@ -332,29 +391,48 @@ define i32 @ctz_v16i1(<16 x i1> %a) {
}
define i32 @ctz_v16i1_poison(<16 x i1> %a) {
-; NONSTREAMING-LABEL: ctz_v16i1_poison:
-; NONSTREAMING: // %bb.0:
-; NONSTREAMING-NEXT: shl v0.16b, v0.16b, #7
-; NONSTREAMING-NEXT: ptrue p0.b, vl16
-; NONSTREAMING-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; NONSTREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; NONSTREAMING-NEXT: cntp x0, p1, p1.b
-; NONSTREAMING-NEXT: ret
;
-; STREAMING-LABEL: ctz_v16i1_poison:
-; STREAMING: // %bb.0:
-; STREAMING-NEXT: lsl z0.b, z0.b, #7
-; STREAMING-NEXT: ptrue p0.b, vl16
-; STREAMING-NEXT: asr z0.b, z0.b, #7
-; STREAMING-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; STREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; STREAMING-NEXT: cntp x0, p1, p1.b
-; STREAMING-NEXT: ret
+; SVE-LABEL: ctz_v16i1_poison:
+; SVE: // %bb.0:
+; SVE-NEXT: shl v0.16b, v0.16b, #7
+; SVE-NEXT: ptrue p0.b, vl16
+; SVE-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SVE-NEXT: brkb p1.b, p0/z, p1.b
+; SVE-NEXT: cntp x0, p1, p1.b
+; SVE-NEXT: ret
+;
+; SME-LABEL: ctz_v16i1_poison:
+; SME: // %bb.0:
+; SME-NEXT: lsl z0.b, z0.b, #7
+; SME-NEXT: ptrue p0.b, vl16
+; SME-NEXT: asr z0.b, z0.b, #7
+; SME-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SME-NEXT: brkb p1.b, p0/z, p1.b
+; SME-NEXT: cntp x0, p1, p1.b
+; SME-NEXT: ret
+;
+; SVE2P2-LABEL: ctz_v16i1_poison:
+; SVE2P2: // %bb.0:
+; SVE2P2-NEXT: shl v0.16b, v0.16b, #7
+; SVE2P2-NEXT: ptrue p0.b, vl16
+; SVE2P2-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SVE2P2-NEXT: firstp x0, p0, p1.b
+; SVE2P2-NEXT: ret
+;
+; SME2P2-LABEL: ctz_v16i1_poison:
+; SME2P2: // %bb.0:
+; SME2P2-NEXT: lsl z0.b, z0.b, #7
+; SME2P2-NEXT: ptrue p0.b, vl16
+; SME2P2-NEXT: asr z0.b, z0.b, #7
+; SME2P2-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SME2P2-NEXT: firstp x0, p0, p1.b
+; SME2P2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 1)
ret i32 %res
}
define i64 @add_i64_ctz_v16i1_poison(<16 x i1> %a, i64 %b) {
+;
; NONSTREAMING-LABEL: add_i64_ctz_v16i1_poison:
; NONSTREAMING: // %bb.0:
; NONSTREAMING-NEXT: shl v0.16b, v0.16b, #7
@@ -402,24 +480,42 @@ define i32 @ctz_v8i1(<8 x i1> %a) {
}
define i32 @ctz_v8i1_poison(<8 x i1> %a) {
-; NONSTREAMING-LABEL: ctz_v8i1_poison:
-; NONSTREAMING: // %bb.0:
-; NONSTREAMING-NEXT: shl v0.8b, v0.8b, #7
-; NONSTREAMING-NEXT: ptrue p0.b, vl8
-; NONSTREAMING-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; NONSTREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; NONSTREAMING-NEXT: cntp x0, p1, p1.b
-; NONSTREAMING-NEXT: ret
;
-; STREAMING-LABEL: ctz_v8i1_poison:
-; STREAMING: // %bb.0:
-; STREAMING-NEXT: lsl z0.b, z0.b, #7
-; STREAMING-NEXT: ptrue p0.b, vl8
-; STREAMING-NEXT: asr z0.b, z0.b, #7
-; STREAMING-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; STREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; STREAMING-NEXT: cntp x0, p1, p1.b
-; STREAMING-NEXT: ret
+; SVE-LABEL: ctz_v8i1_poison:
+; SVE: // %bb.0:
+; SVE-NEXT: shl v0.8b, v0.8b, #7
+; SVE-NEXT: ptrue p0.b, vl8
+; SVE-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SVE-NEXT: brkb p1.b, p0/z, p1.b
+; SVE-NEXT: cntp x0, p1, p1.b
+; SVE-NEXT: ret
+;
+; SME-LABEL: ctz_v8i1_poison:
+; SME: // %bb.0:
+; SME-NEXT: lsl z0.b, z0.b, #7
+; SME-NEXT: ptrue p0.b, vl8
+; SME-NEXT: asr z0.b, z0.b, #7
+; SME-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SME-NEXT: brkb p1.b, p0/z, p1.b
+; SME-NEXT: cntp x0, p1, p1.b
+; SME-NEXT: ret
+;
+; SVE2P2-LABEL: ctz_v8i1_poison:
+; SVE2P2: // %bb.0:
+; SVE2P2-NEXT: shl v0.8b, v0.8b, #7
+; SVE2P2-NEXT: ptrue p0.b, vl8
+; SVE2P2-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SVE2P2-NEXT: firstp x0, p0, p1.b
+; SVE2P2-NEXT: ret
+;
+; SME2P2-LABEL: ctz_v8i1_poison:
+; SME2P2: // %bb.0:
+; SME2P2-NEXT: lsl z0.b, z0.b, #7
+; SME2P2-NEXT: ptrue p0.b, vl8
+; SME2P2-NEXT: asr z0.b, z0.b, #7
+; SME2P2-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; SME2P2-NEXT: firstp x0, p0, p1.b
+; SME2P2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> %a, i1 1)
ret i32 %res
}
@@ -448,24 +544,42 @@ define i32 @ctz_v4i1(<4 x i1> %a) {
}
define i32 @ctz_v4i1_poison(<4 x i1> %a) {
-; NONSTREAMING-LABEL: ctz_v4i1_poison:
-; NONSTREAMING: // %bb.0:
-; NONSTREAMING-NEXT: shl v0.4h, v0.4h, #15
-; NONSTREAMING-NEXT: ptrue p0.h, vl4
-; NONSTREAMING-NEXT: cmpne p1.h, p0/z, z0.h, #0
-; NONSTREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; NONSTREAMING-NEXT: cntp x0, p1, p1.h
-; NONSTREAMING-NEXT: ret
;
-; STREAMING-LABEL: ctz_v4i1_poison:
-; STREAMING: // %bb.0:
-; STREAMING-NEXT: lsl z0.h, z0.h, #15
-; STREAMING-NEXT: ptrue p0.h, vl4
-; STREAMING-NEXT: asr z0.h, z0.h, #15
-; STREAMING-NEXT: cmpne p1.h, p0/z, z0.h, #0
-; STREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; STREAMING-NEXT: cntp x0, p1, p1.h
-; STREAMING-NEXT: ret
+; SVE-LABEL: ctz_v4i1_poison:
+; SVE: // %bb.0:
+; SVE-NEXT: shl v0.4h, v0.4h, #15
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; SVE-NEXT: brkb p1.b, p0/z, p1.b
+; SVE-NEXT: cntp x0, p1, p1.h
+; SVE-NEXT: ret
+;
+; SME-LABEL: ctz_v4i1_poison:
+; SME: // %bb.0:
+; SME-NEXT: lsl z0.h, z0.h, #15
+; SME-NEXT: ptrue p0.h, vl4
+; SME-NEXT: asr z0.h, z0.h, #15
+; SME-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; SME-NEXT: brkb p1.b, p0/z, p1.b
+; SME-NEXT: cntp x0, p1, p1.h
+; SME-NEXT: ret
+;
+; SVE2P2-LABEL: ctz_v4i1_poison:
+; SVE2P2: // %bb.0:
+; SVE2P2-NEXT: shl v0.4h, v0.4h, #15
+; SVE2P2-NEXT: ptrue p0.h, vl4
+; SVE2P2-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; SVE2P2-NEXT: firstp x0, p0, p1.h
+; SVE2P2-NEXT: ret
+;
+; SME2P2-LABEL: ctz_v4i1_poison:
+; SME2P2: // %bb.0:
+; SME2P2-NEXT: lsl z0.h, z0.h, #15
+; SME2P2-NEXT: ptrue p0.h, vl4
+; SME2P2-NEXT: asr z0.h, z0.h, #15
+; SME2P2-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; SME2P2-NEXT: firstp x0, p0, p1.h
+; SME2P2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> %a, i1 1)
ret i32 %res
}
@@ -494,24 +608,42 @@ define i32 @ctz_v2i1(<2 x i1> %a) {
}
define i32 @ctz_v2i1_poison(<2 x i1> %a) {
-; NONSTREAMING-LABEL: ctz_v2i1_poison:
-; NONSTREAMING: // %bb.0:
-; NONSTREAMING-NEXT: shl v0.2s, v0.2s, #31
-; NONSTREAMING-NEXT: ptrue p0.s, vl2
-; NONSTREAMING-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; NONSTREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; NONSTREAMING-NEXT: cntp x0, p1, p1.s
-; NONSTREAMING-NEXT: ret
;
-; STREAMING-LABEL: ctz_v2i1_poison:
-; STREAMING: // %bb.0:
-; STREAMING-NEXT: lsl z0.s, z0.s, #31
-; STREAMING-NEXT: ptrue p0.s, vl2
-; STREAMING-NEXT: asr z0.s, z0.s, #31
-; STREAMING-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; STREAMING-NEXT: brkb p1.b, p0/z, p1.b
-; STREAMING-NEXT: cntp x0, p1, p1.s
-; STREAMING-NEXT: ret
+; SVE-LABEL: ctz_v2i1_poison:
+; SVE: // %bb.0:
+; SVE-NEXT: shl v0.2s, v0.2s, #31
+; SVE-NEXT: ptrue p0.s, vl2
+; SVE-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; SVE-NEXT: brkb p1.b, p0/z, p1.b
+; SVE-NEXT: cntp x0, p1, p1.s
+; SVE-NEXT: ret
+;
+; SME-LABEL: ctz_v2i1_poison:
+; SME: // %bb.0:
+; SME-NEXT: lsl z0.s, z0.s, #31
+; SME-NEXT: ptrue p0.s, vl2
+; SME-NEXT: asr z0.s, z0.s, #31
+; SME-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; SME-NEXT: brkb p1.b, p0/z, p1.b
+; SME-NEXT: cntp x0, p1, p1.s
+; SME-NEXT: ret
+;
+; SVE2P2-LABEL: ctz_v2i1_poison:
+; SVE2P2: // %bb.0:
+; SVE2P2-NEXT: shl v0.2s, v0.2s, #31
+; SVE2P2-NEXT: ptrue p0.s, vl2
+; SVE2P2-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; SVE2P2-NEXT: firstp x0, p0, p1.s
+; SVE2P2-NEXT: ret
+;
+; SME2P2-LABEL: ctz_v2i1_poison:
+; SME2P2: // %bb.0:
+; SME2P2-NEXT: lsl z0.s, z0.s, #31
+; SME2P2-NEXT: ptrue p0.s, vl2
+; SME2P2-NEXT: asr z0.s, z0.s, #31
+; SME2P2-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; SME2P2-NEXT: firstp x0, p0, p1.s
+; SME2P2-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> %a, i1 1)
ret i32 %res
}
More information about the llvm-commits
mailing list