[llvm] [AArch64] NFC: Replace CTTZ_ELTS predicate pattern with predicate. (PR #179909)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 5 05:47:07 PST 2026
https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/179909
>From c7b1d037f91987344883047d50b2079cdc483294 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 5 Feb 2026 10:15:11 +0000
Subject: [PATCH 1/3] [AArch64] NFC: Replace CTTZ_ELTS predicate pattern with
predicate.
This also swaps the general predicate and source operand,
to match other nodes.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 17 +++---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 2 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 56 +++++++++----------
3 files changed, 38 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 88836d6e167b8..128340d921b03 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3566,6 +3566,8 @@ static SDValue convertFixedMaskToScalableVector(SDValue Mask,
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
EVT VT);
+static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
+ EVT VT);
/// isZerosVector - Check whether SDNode N is a zero-filled vector.
static bool isZerosVector(const SDNode *N) {
@@ -6051,9 +6053,8 @@ static SDValue optimizeBrk(SDNode *N, SelectionDAG &DAG) {
if (Upper.getOpcode() != AArch64ISD::CTTZ_ELTS || !VT.isScalableVector())
return SDValue();
- SDValue Mask = Upper->getOperand(0);
- const APInt &PgPattern = Upper.getConstantOperandAPInt(1);
- SDValue Pg = getPTrue(DAG, DL, VT, PgPattern.getZExtValue());
+ SDValue Pg = Upper->getOperand(0);
+ SDValue Mask = Upper->getOperand(1);
// brk{a,b} only support .b forms, so cast to make sure all our p regs match.
Pg = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Pg);
@@ -6947,7 +6948,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
assert(VT.getVectorElementType() == MVT::i1 && "Expected MVT::i1");
// Default to all for scalable vectors
- unsigned PgPattern = AArch64SVEPredPattern::all;
+ SDValue Pg;
if (VT.isFixedLengthVector()) {
// We can use SVE instructions to lower this intrinsic by first creating
// an SVE predicate register mask from the fixed-width vector.
@@ -6955,12 +6956,12 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, CttzOp);
CttzOp = convertFixedMaskToScalableVector(Mask, DAG);
// Override with a VLx.
- PgPattern = *getSVEPredPatternFromNumElements(VT.getVectorNumElements());
- }
+ Pg = getPredicateForFixedLengthVector(DAG, DL, NewVT);
+ } else
+ Pg = DAG.getConstant(1, DL, VT);
- SDValue Pattern = DAG.getTargetConstant(PgPattern, DL, MVT::i32);
SDValue NewCttzElts =
- DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, CttzOp, Pattern);
+ DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, Pg, CttzOp);
return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
}
case Intrinsic::experimental_vector_match: {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index bd67f97bc0e03..f6393c227bc72 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1225,7 +1225,7 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
(int_aarch64_sve_rshrnb node:$rs, node:$i)]>;
def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 2,
- [SDTCisInt<0>, SDTCisVec<1>, SDTCisInt<2>]>, []>;
+ [SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>]>, []>;
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 51b260755576a..83a6c7959c6ea 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2273,21 +2273,21 @@ let Predicates = [HasSVE_or_SME] in {
defm CNTD_XPiI : sve_int_count<0b110, "cntd", int_aarch64_sve_cntd>;
defm CNTP_XPP : sve_int_pcount_pred<0b000, "cntp", int_aarch64_sve_cntp>;
- def : Pat<(i64 (AArch64CttzElts nxv16i1:$Op1, sve_pred_enum:$Pat)),
- (CNTP_XPP_B (BRKB_PPzP (PTRUE_B $Pat), PPR:$Op1),
- (BRKB_PPzP (PTRUE_B $Pat), PPR:$Op1))>;
+ def : Pat<(i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op1)),
+ (CNTP_XPP_B (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
- def : Pat<(i64 (AArch64CttzElts nxv8i1:$Op1, sve_pred_enum:$Pat)),
- (CNTP_XPP_H (BRKB_PPzP (PTRUE_H $Pat), PPR:$Op1),
- (BRKB_PPzP (PTRUE_H $Pat), PPR:$Op1))>;
+ def : Pat<(i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op1)),
+ (CNTP_XPP_H (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
- def : Pat<(i64 (AArch64CttzElts nxv4i1:$Op1, sve_pred_enum:$Pat)),
- (CNTP_XPP_S (BRKB_PPzP (PTRUE_S $Pat), PPR:$Op1),
- (BRKB_PPzP (PTRUE_S $Pat), PPR:$Op1))>;
+ def : Pat<(i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op1)),
+ (CNTP_XPP_S (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
- def : Pat<(i64 (AArch64CttzElts nxv2i1:$Op1, sve_pred_enum:$Pat)),
- (CNTP_XPP_D (BRKB_PPzP (PTRUE_D $Pat), PPR:$Op1),
- (BRKB_PPzP (PTRUE_D $Pat), PPR:$Op1))>;
+ def : Pat<(i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op1)),
+ (CNTP_XPP_D (BRKB_PPzP PPR:$Pg, PPR:$Op1),
+ (BRKB_PPzP PPR:$Pg, PPR:$Op1))>;
}
defm INCB_XPiI : sve_int_pred_pattern_a<0b000, "incb", add, int_aarch64_sve_cntb>;
@@ -2373,35 +2373,35 @@ let Predicates = [HasSVE_or_SME] in {
defm INCP_ZP : sve_int_count_v<0b10000, "incp">;
defm DECP_ZP : sve_int_count_v<0b10100, "decp">;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv16i1:$Op2, sve_pred_enum:$Pat)))),
- (INCP_XP_B (BRKB_PPzP (PTRUE_B $Pat), PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op2)))),
+ (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv16i1:$Op2, sve_pred_enum:$Pat))))),
- (EXTRACT_SUBREG (INCP_XP_B (BRKB_PPzP (PTRUE_B $Pat), PPR:$Op2),
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv16i1:$Pg, nxv16i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_B (BRKB_PPzP PPR:$Pg, PPR:$Op2),
(INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv8i1:$Op2, sve_pred_enum:$Pat)))),
- (INCP_XP_H (BRKB_PPzP (PTRUE_H $Pat), PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op2)))),
+ (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv8i1:$Op2, sve_pred_enum:$Pat))))),
- (EXTRACT_SUBREG (INCP_XP_H (BRKB_PPzP (PTRUE_H $Pat), PPR:$Op2),
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv8i1:$Pg, nxv8i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_H (BRKB_PPzP PPR:$Pg, PPR:$Op2),
(INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv4i1:$Op2, sve_pred_enum:$Pat)))),
- (INCP_XP_S (BRKB_PPzP (PTRUE_S $Pat), PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op2)))),
+ (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv4i1:$Op2, sve_pred_enum:$Pat))))),
- (EXTRACT_SUBREG (INCP_XP_S (BRKB_PPzP (PTRUE_S $Pat), PPR:$Op2),
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv4i1:$Pg, nxv4i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_S (BRKB_PPzP PPR:$Pg, PPR:$Op2),
(INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
sub_32)>;
- def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv2i1:$Op2, sve_pred_enum:$Pat)))),
- (INCP_XP_D (BRKB_PPzP (PTRUE_D $Pat), PPR:$Op2), GPR64:$Op1)>;
+ def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op2)))),
+ (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2), GPR64:$Op1)>;
- def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv2i1:$Op2, sve_pred_enum:$Pat))))),
- (EXTRACT_SUBREG (INCP_XP_D (BRKB_PPzP (PTRUE_D $Pat), PPR:$Op2),
+ def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv2i1:$Pg, nxv2i1:$Op2))))),
+ (EXTRACT_SUBREG (INCP_XP_D (BRKB_PPzP PPR:$Pg, PPR:$Op2),
(INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
sub_32)>;
>From 63f7a7ad2af68aa14bcc26b832a7a068ada4c87e Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 5 Feb 2026 13:39:35 +0000
Subject: [PATCH 2/3] Address comments
---
.../Target/AArch64/AArch64ISelLowering.cpp | 19 ++++++++-----------
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 ++-
2 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 128340d921b03..7f29b50ded32e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3566,8 +3566,9 @@ static SDValue convertFixedMaskToScalableVector(SDValue Mask,
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
EVT VT);
-static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
- EVT VT);
+static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL,
+ EVT VT);
+static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG);
/// isZerosVector - Check whether SDNode N is a zero-filled vector.
static bool isZerosVector(const SDNode *N) {
@@ -6057,7 +6058,7 @@ static SDValue optimizeBrk(SDNode *N, SelectionDAG &DAG) {
SDValue Mask = Upper->getOperand(1);
// brk{a,b} only support .b forms, so cast to make sure all our p regs match.
- Pg = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Pg);
+ Pg = getSVEPredicateBitCast(MVT::nxv16i1, Pg, DAG);
SDValue MaskR =
DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Mask);
SDValue ID = DAG.getTargetConstant(BrkID, DL, MVT::i64);
@@ -6947,19 +6948,15 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
EVT VT = CttzOp.getValueType();
assert(VT.getVectorElementType() == MVT::i1 && "Expected MVT::i1");
- // Default to all for scalable vectors
- SDValue Pg;
if (VT.isFixedLengthVector()) {
// We can use SVE instructions to lower this intrinsic by first creating
// an SVE predicate register mask from the fixed-width vector.
- EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
- SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, CttzOp);
+ VT = getTypeToTransformTo(*DAG.getContext(), VT);
+ SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, CttzOp);
CttzOp = convertFixedMaskToScalableVector(Mask, DAG);
- // Override with a VLx.
- Pg = getPredicateForFixedLengthVector(DAG, DL, NewVT);
- } else
- Pg = DAG.getConstant(1, DL, VT);
+ }
+ SDValue Pg = getPredicateForVector(DAG, DL, VT);
SDValue NewCttzElts =
DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, Pg, CttzOp);
return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f6393c227bc72..6c4e7ec15a163 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1224,8 +1224,9 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
[(AArch64rshrnb node:$rs, node:$i),
(int_aarch64_sve_rshrnb node:$rs, node:$i)]>;
+// res = cttz.elts(pg, op)
def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 2,
- [SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>]>, []>;
+ [SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
>From 68bc233190c10ba168a3c6bcd5801d6b8612d226 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 5 Feb 2026 13:46:12 +0000
Subject: [PATCH 3/3] remove redundant declare of getPredicateForVector
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7f29b50ded32e..94216f6572f0a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3566,8 +3566,6 @@ static SDValue convertFixedMaskToScalableVector(SDValue Mask,
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
EVT VT);
-static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL,
- EVT VT);
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG);
/// isZerosVector - Check whether SDNode N is a zero-filled vector.
More information about the llvm-commits
mailing list