[llvm] [AArch64][SVE] Expose flags result of predicate-as-counter whiles (NFC) (PR #202976)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 06:53:34 PDT 2026
https://github.com/MacDue created https://github.com/llvm/llvm-project/pull/202976
For predicate-as-counter there is not "ptest" instruction. So unlike for the SVE1 while instructions, we can't fold
`extract_vector_elt(pext(whilelo, 0), 0)` to PTEST_FIRST_ACTIVE, then rely on the ptest elimination to remove redundant tests.
This patch instead adds new ISD nodes for predicate-as-counter while instructions that return both the predicate and the status flags. This will allow us to DAG combine `extract_vector_elt(pext(whilelo, 0), 0)` to `CSET (whilelo, first)`. A further DAG combine could also fold the `CSET` into a BR_CC (if it has one use).
>From d2119127049e5d0c301cdf8cd5183fe46bb20732 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 10 Jun 2026 13:07:26 +0000
Subject: [PATCH] [AArch64][SVE] Expose flags result of predicate-as-counter
whiles (NFC)
For predicate-as-counter there is not "ptest" instruction. So unlike
for the SVE1 while instructions, we can't fold
`extract_vector_elt(pext(whilelo, 0), 0)` to PTEST_FIRST_ACTIVE, then
rely on the ptest elimination to remove redundant tests.
This patch instead adds new ISD nodes for predicate-as-counter while
instructions that return both the predicate and the status flags. This
will allow us to DAG combine `extract_vector_elt(pext(whilelo, 0), 0)`
to `CSET (whilelo, first)`. A further DAG combine could also fold the
`CSET` into a BR_CC (if it has one use).
---
.../Target/AArch64/AArch64ISelLowering.cpp | 140 ++++++++++++++++++
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 33 ++++-
llvm/lib/Target/AArch64/SVEInstrFormats.td | 30 +---
3 files changed, 172 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 57a2d73e00f57..b269058ac50b0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -24188,6 +24188,113 @@ tryCombineFADDReductionWithZero(SDNode *N, SelectionDAG &DAG,
return Elts[0];
}
+/// Returns the element size of a predicate-as-counter while intrinsic.
+static unsigned getPredCounterWhileElementSize(Intrinsic::ID IID) {
+ switch (IID) {
+ case Intrinsic::aarch64_sve_whilege_c8:
+ case Intrinsic::aarch64_sve_whilegt_c8:
+ case Intrinsic::aarch64_sve_whilelt_c8:
+ case Intrinsic::aarch64_sve_whilele_c8:
+ case Intrinsic::aarch64_sve_whilehs_c8:
+ case Intrinsic::aarch64_sve_whilehi_c8:
+ case Intrinsic::aarch64_sve_whilelo_c8:
+ case Intrinsic::aarch64_sve_whilels_c8:
+ return 8;
+ case Intrinsic::aarch64_sve_whilege_c16:
+ case Intrinsic::aarch64_sve_whilegt_c16:
+ case Intrinsic::aarch64_sve_whilelt_c16:
+ case Intrinsic::aarch64_sve_whilele_c16:
+ case Intrinsic::aarch64_sve_whilehs_c16:
+ case Intrinsic::aarch64_sve_whilehi_c16:
+ case Intrinsic::aarch64_sve_whilelo_c16:
+ case Intrinsic::aarch64_sve_whilels_c16:
+ return 16;
+ case Intrinsic::aarch64_sve_whilege_c32:
+ case Intrinsic::aarch64_sve_whilegt_c32:
+ case Intrinsic::aarch64_sve_whilelt_c32:
+ case Intrinsic::aarch64_sve_whilele_c32:
+ case Intrinsic::aarch64_sve_whilehs_c32:
+ case Intrinsic::aarch64_sve_whilehi_c32:
+ case Intrinsic::aarch64_sve_whilelo_c32:
+ case Intrinsic::aarch64_sve_whilels_c32:
+ return 32;
+ case Intrinsic::aarch64_sve_whilege_c64:
+ case Intrinsic::aarch64_sve_whilegt_c64:
+ case Intrinsic::aarch64_sve_whilelt_c64:
+ case Intrinsic::aarch64_sve_whilele_c64:
+ case Intrinsic::aarch64_sve_whilehs_c64:
+ case Intrinsic::aarch64_sve_whilehi_c64:
+ case Intrinsic::aarch64_sve_whilelo_c64:
+ case Intrinsic::aarch64_sve_whilels_c64:
+ return 64;
+ default:
+ llvm_unreachable("Unexpected IID");
+ }
+}
+
+/// Returns the ISD opcode for a predicate-as-counter while intrinsic.
+static unsigned getPredCounterWhileOpcode(Intrinsic::ID IID) {
+ switch (IID) {
+ case Intrinsic::aarch64_sve_whilege_c8:
+ case Intrinsic::aarch64_sve_whilege_c16:
+ case Intrinsic::aarch64_sve_whilege_c32:
+ case Intrinsic::aarch64_sve_whilege_c64:
+ return AArch64ISD::WHILEGE_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilegt_c8:
+ case Intrinsic::aarch64_sve_whilegt_c16:
+ case Intrinsic::aarch64_sve_whilegt_c32:
+ case Intrinsic::aarch64_sve_whilegt_c64:
+ return AArch64ISD::WHILEGT_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilelt_c8:
+ case Intrinsic::aarch64_sve_whilelt_c16:
+ case Intrinsic::aarch64_sve_whilelt_c32:
+ case Intrinsic::aarch64_sve_whilelt_c64:
+ return AArch64ISD::WHILELT_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilele_c8:
+ case Intrinsic::aarch64_sve_whilele_c16:
+ case Intrinsic::aarch64_sve_whilele_c32:
+ case Intrinsic::aarch64_sve_whilele_c64:
+ return AArch64ISD::WHILELE_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilehs_c8:
+ case Intrinsic::aarch64_sve_whilehs_c16:
+ case Intrinsic::aarch64_sve_whilehs_c32:
+ case Intrinsic::aarch64_sve_whilehs_c64:
+ return AArch64ISD::WHILEHS_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilehi_c8:
+ case Intrinsic::aarch64_sve_whilehi_c16:
+ case Intrinsic::aarch64_sve_whilehi_c32:
+ case Intrinsic::aarch64_sve_whilehi_c64:
+ return AArch64ISD::WHILEHI_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilelo_c8:
+ case Intrinsic::aarch64_sve_whilelo_c16:
+ case Intrinsic::aarch64_sve_whilelo_c32:
+ case Intrinsic::aarch64_sve_whilelo_c64:
+ return AArch64ISD::WHILELO_PRED_COUNTER;
+ case Intrinsic::aarch64_sve_whilels_c8:
+ case Intrinsic::aarch64_sve_whilels_c16:
+ case Intrinsic::aarch64_sve_whilels_c32:
+ case Intrinsic::aarch64_sve_whilels_c64:
+ return AArch64ISD::WHILELS_PRED_COUNTER;
+ default:
+ llvm_unreachable("Unexpected IID");
+ }
+}
+
+/// Lower a predicate-as-counter while intrinsic to an AArch64ISD node.
+static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
+ SelectionDAG &DAG) {
+ SDLoc DL(N);
+ unsigned ElementSizeInBits = getPredCounterWhileElementSize(IID);
+ unsigned VectorScale = N->getConstantOperandVal(3);
+ assert(VectorScale == 2 || VectorScale == 4);
+ return DAG.getNode(
+ getPredCounterWhileOpcode(IID), DL,
+ DAG.getVTList(MVT::aarch64svcount, FlagsVT), N->getOperand(1),
+ N->getOperand(2),
+ DAG.getTargetConstant(VectorScale == 2 ? 0 : 1, DL, MVT::i32),
+ DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -24525,6 +24632,39 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bsl2n:
case Intrinsic::aarch64_sve_nbsl:
return combineSVEBitSel(IID, N, DAG);
+ case Intrinsic::aarch64_sve_whilege_c8:
+ case Intrinsic::aarch64_sve_whilege_c16:
+ case Intrinsic::aarch64_sve_whilege_c32:
+ case Intrinsic::aarch64_sve_whilege_c64:
+ case Intrinsic::aarch64_sve_whilegt_c8:
+ case Intrinsic::aarch64_sve_whilegt_c16:
+ case Intrinsic::aarch64_sve_whilegt_c32:
+ case Intrinsic::aarch64_sve_whilegt_c64:
+ case Intrinsic::aarch64_sve_whilelt_c8:
+ case Intrinsic::aarch64_sve_whilelt_c16:
+ case Intrinsic::aarch64_sve_whilelt_c32:
+ case Intrinsic::aarch64_sve_whilelt_c64:
+ case Intrinsic::aarch64_sve_whilele_c8:
+ case Intrinsic::aarch64_sve_whilele_c16:
+ case Intrinsic::aarch64_sve_whilele_c32:
+ case Intrinsic::aarch64_sve_whilele_c64:
+ case Intrinsic::aarch64_sve_whilehs_c8:
+ case Intrinsic::aarch64_sve_whilehs_c16:
+ case Intrinsic::aarch64_sve_whilehs_c32:
+ case Intrinsic::aarch64_sve_whilehs_c64:
+ case Intrinsic::aarch64_sve_whilehi_c8:
+ case Intrinsic::aarch64_sve_whilehi_c16:
+ case Intrinsic::aarch64_sve_whilehi_c32:
+ case Intrinsic::aarch64_sve_whilehi_c64:
+ case Intrinsic::aarch64_sve_whilelo_c8:
+ case Intrinsic::aarch64_sve_whilelo_c16:
+ case Intrinsic::aarch64_sve_whilelo_c32:
+ case Intrinsic::aarch64_sve_whilelo_c64:
+ case Intrinsic::aarch64_sve_whilels_c8:
+ case Intrinsic::aarch64_sve_whilels_c16:
+ case Intrinsic::aarch64_sve_whilels_c32:
+ case Intrinsic::aarch64_sve_whilels_c64:
+ return lowerPredCounterWhile(IID, N, DAG);
}
return SDValue();
}
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 0cc788d12bae0..feebb7ce89b1a 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -39,6 +39,7 @@
// _mt <=> _MERGE_PASSTHRU
// _z <=> _MERGE_ZERO
// _p <=> _PRED
+// _pn <=> _PRED_COUNTER
//
// Given the context of this file, it is not strictly necessary to use _p to
// distinguish predicated from unpredicated nodes given that most SVE
@@ -73,6 +74,22 @@ def SDT_AArch64_LD1Replicate : SDTypeProfile<1, 2, [
def AArch64ld1rq_z : SDNode<"AArch64ISD::LD1RQ_MERGE_ZERO", SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
def AArch64ld1ro_z : SDNode<"AArch64ISD::LD1RO_MERGE_ZERO", SDT_AArch64_LD1Replicate, [SDNPHasChain, SDNPMayLoad]>;
+def SDT_AArch64_PredCounterWhile : SDTypeProfile<2, 4, [
+ SDTCisVT<0, aarch64svcount>, SDTCisInt<1>,
+ SDTCisInt<2>, SDTCisInt<3>,
+ SDTCisInt<4>, SDTCisInt<5>,
+ SDTCisVT<1, FlagsVT>
+]>;
+
+def AArch64whilege_pn_flag : SDNode<"AArch64ISD::WHILEGE_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilegt_pn_flag : SDNode<"AArch64ISD::WHILEGT_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilelt_pn_flag : SDNode<"AArch64ISD::WHILELT_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilele_pn_flag : SDNode<"AArch64ISD::WHILELE_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilehs_pn_flag : SDNode<"AArch64ISD::WHILEHS_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilehi_pn_flag : SDNode<"AArch64ISD::WHILEHI_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilelo_pn_flag : SDNode<"AArch64ISD::WHILELO_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+def AArch64whilels_pn_flag : SDNode<"AArch64ISD::WHILELS_PRED_COUNTER", SDT_AArch64_PredCounterWhile>;
+
// Gather loads - node definitions
//
def SDT_AArch64_GATHER_SV : SDTypeProfile<1, 4, [
@@ -4607,14 +4624,14 @@ defm : store_pn_x4<nxv8bf16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM>;
defm : store_pn_x4<nxv4f32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM>;
defm : store_pn_x4<nxv2f64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM>;
-defm WHILEGE_CXX : sve2p1_int_while_rr_pn<"whilege", 0b000>;
-defm WHILEGT_CXX : sve2p1_int_while_rr_pn<"whilegt", 0b001>;
-defm WHILELT_CXX : sve2p1_int_while_rr_pn<"whilelt", 0b010>;
-defm WHILELE_CXX : sve2p1_int_while_rr_pn<"whilele", 0b011>;
-defm WHILEHS_CXX : sve2p1_int_while_rr_pn<"whilehs", 0b100>;
-defm WHILEHI_CXX : sve2p1_int_while_rr_pn<"whilehi", 0b101>;
-defm WHILELO_CXX : sve2p1_int_while_rr_pn<"whilelo", 0b110>;
-defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111>;
+defm WHILEGE_CXX : sve2p1_int_while_rr_pn<"whilege", 0b000, AArch64whilege_pn_flag>;
+defm WHILEGT_CXX : sve2p1_int_while_rr_pn<"whilegt", 0b001, AArch64whilegt_pn_flag>;
+defm WHILELT_CXX : sve2p1_int_while_rr_pn<"whilelt", 0b010, AArch64whilelt_pn_flag>;
+defm WHILELE_CXX : sve2p1_int_while_rr_pn<"whilele", 0b011, AArch64whilele_pn_flag>;
+defm WHILEHS_CXX : sve2p1_int_while_rr_pn<"whilehs", 0b100, AArch64whilehs_pn_flag>;
+defm WHILEHI_CXX : sve2p1_int_while_rr_pn<"whilehi", 0b101, AArch64whilehi_pn_flag>;
+defm WHILELO_CXX : sve2p1_int_while_rr_pn<"whilelo", 0b110, AArch64whilelo_pn_flag>;
+defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111, AArch64whilels_pn_flag>;
} // End HasSVE2p1_or_StreamingSME2
let Predicates = [HasSVE_or_SME] in {
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 040962e801604..54e960719a136 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -617,12 +617,6 @@ multiclass SVE2p1_Cntp_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
def : Pat<(vtd (op vt1:$Op1, (i32 4))), (inst $Op1, 1)>;
}
-multiclass SVE2p1_While_PN_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
- Instruction inst> {
- def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 2))), (inst $Op1, $Op2, 0)>;
- def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 4))), (inst $Op1, $Op2, 1)>;
-}
-
class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, Operand ImmTy,
Instruction inst>
@@ -10495,13 +10489,12 @@ multiclass sve2p1_pcount_pn<string mnemonic, bits<3> opc> {
defm : SVE2p1_Cntp_Pat<i64, int_aarch64_sve_cntp_c64, aarch64svcount, !cast<Instruction>(NAME # _D)>;
}
-
// SVE integer compare scalar count and limit (predicate-as-counter)
-class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc,
+class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc, SDNode OpNode, int ElTy,
PNRP8to15RegOp pnrty>
: I<(outs pnrty:$PNd), (ins GPR64:$Rn, GPR64:$Rm, sve_vec_len_specifier_enum:$vl),
mnemonic, "\t$PNd, $Rn, $Rm, $vl",
- "", []>, Sched<[]> {
+ "", [(set pnrty:$PNd, (OpNode i64:$Rn, i64:$Rm, (sve_vec_len_specifier_enum:$vl), (i64 ElTy)))]>, Sched<[]> {
bits<3> PNd;
bits<5> Rn;
bits<1> vl;
@@ -10524,20 +10517,11 @@ class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc,
}
-multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> {
- def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, PNR8_p8to15>;
- def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>;
- def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>;
- def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>;
-
- defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c8"),
- i64, !cast<Instruction>(NAME # _B)>;
- defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c16"),
- i64, !cast<Instruction>(NAME # _H)>;
- defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c32"),
- i64, !cast<Instruction>(NAME # _S)>;
- defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c64"),
- i64, !cast<Instruction>(NAME # _D)>;
+multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc, SDNode OpNode> {
+ def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, OpNode, 8, PNR8_p8to15>;
+ def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, OpNode, 16, PNR16_p8to15>;
+ def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, OpNode, 32, PNR32_p8to15>;
+ def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, OpNode, 64, PNR64_p8to15>;
}
More information about the llvm-commits
mailing list