[llvm] [AArch64] Lower fixed find_last_active to LASTP with SVE2.2 (PR #222534)
Vimal Patel via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 06:59:22 PDT 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/222534
>From 0322b3e097d06392834cffdfbc56adc85562c58f Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Tue, 8 Sep 2026 12:55:19 +0000
Subject: [PATCH] [AArch64] Lower fixed find_last_active to LASTP with SVE2.2
Represent fixed-length VECTOR_FIND_LAST_ACTIVE with a target node that
preserves the original mask and returns -1 for an empty mask. This lets
DAG combining reuse the result instead of emitting a separate UMAXV.
Convert the mask to an SVE predicate immediately before instruction
selection, then select LASTP through TableGen. Leave scalable forms
legal so existing LASTB/CLASTB combines continue to apply.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 6 +
.../Target/AArch64/AArch64ISelLowering.cpp | 114 +++++++
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 10 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 12 +
.../AArch64/AArch64SelectionDAGInfo.cpp | 1 +
.../AArch64/vector-extract-last-active.ll | 279 ++++++++++++++++--
7 files changed, 402 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 531478517ef9a..b642f6715574b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -8296,6 +8296,12 @@ void AArch64DAGToDAGISel::PreprocessISelDAG() {
SDValue Result;
switch (N.getOpcode()) {
+ case AArch64ISD::FIND_LAST_ACTIVE: {
+ const auto *TLI = static_cast<const AArch64TargetLowering *>(
+ Subtarget->getTargetLowering());
+ Result = TLI->expandFixedLengthFindLastActive(SDValue(&N, 0), *CurDAG);
+ break;
+ }
case ISD::SCALAR_TO_VECTOR: {
EVT ScalarTy = N.getValueType(0).getVectorElementType();
if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c51683b78cb54..2f9f6998bd8ba 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2273,6 +2273,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ // Mark the fixed-length vector container types for custom lowering.
+ // These variants lower to lastp. The scalable variants lower to
+ // lastb/clastb.
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32, MVT::v1i64, MVT::v2i64})
+ setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Custom);
+
} else {
// Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
// which is not supported with for compact (with only +sve).
@@ -8257,6 +8265,46 @@ SDValue AArch64TargetLowering::LowerVECTOR_COMPRESS(SDValue Op,
Passthru);
}
+SDValue
+AArch64TargetLowering::LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ SDValue Find = DAG.getNode(AArch64ISD::FIND_LAST_ACTIVE, DL,
+ Op.getValueType(), Op.getOperand(0));
+ SDValue AllInactive = DAG.getSignedConstant(-1, DL, Find.getValueType());
+ SDValue Zero = DAG.getSignedConstant(0, DL, Find.getValueType());
+
+ // Keep the index in range because scalable vector extraction may spill the
+ // vector and access the selected element through an unconditional load.
+ return DAG.getSelectCC(DL, Find, AllInactive, Zero, Find, ISD::SETEQ);
+}
+
+SDValue AArch64TargetLowering::expandFixedLengthFindLastActive(
+ SDValue Op, SelectionDAG &DAG) const {
+ assert(Op.getOpcode() == AArch64ISD::FIND_LAST_ACTIVE);
+ SDLoc DL(Op);
+ SDValue Mask = Op.getOperand(0);
+ EVT MaskVT = Mask.getValueType();
+ assert(MaskVT.isFixedLengthVector());
+
+ // Type legalization can represent a boolean mask by moving its low bit to
+ // the sign bit and sign-extending it across each element. Predicate
+ // conversion only tests whether an element is nonzero, so the sign extension
+ // is unnecessary.
+ if (Mask.getOpcode() == AArch64ISD::VASHR &&
+ Mask.getOperand(0).getOpcode() == AArch64ISD::VSHL) {
+ SDValue Shifted = Mask.getOperand(0);
+ unsigned TopBit = MaskVT.getScalarSizeInBits() - 1;
+ if (Mask.getConstantOperandVal(1) == TopBit &&
+ Shifted.getConstantOperandVal(1) == TopBit)
+ Mask = Shifted;
+ }
+
+ SDValue Pg = getPredicateForVector(DAG, DL, MaskVT);
+ Mask = convertFixedMaskToScalableVector(Mask, DAG);
+ return DAG.getNode(AArch64ISD::LASTP, DL, Op.getValueType(), Pg, Mask);
+}
+
SDValue AArch64TargetLowering::LowerSMULFIXSAT(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -8967,6 +9015,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVSCALE(Op, DAG);
case ISD::VECTOR_COMPRESS:
return LowerVECTOR_COMPRESS(Op, DAG);
+ case ISD::VECTOR_FIND_LAST_ACTIVE:
+ return LowerVECTOR_FIND_LAST_ACTIVE(Op, DAG);
case ISD::ANY_EXTEND:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
@@ -30046,6 +30096,67 @@ static SDValue performVSelectCombine(SDNode *N,
IfTrue, IfFalse);
}
+static SDValue
+performFindLastActiveSelectCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ if (DCI.isBeforeLegalize())
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ // Replace the reduction used to detect an empty fixed-length mask with the
+ // sentinel returned by FIND_LAST_ACTIVE. Keep the normalized index used by
+ // the extract because it can become an unconditional indexed load.
+ //
+ // select (and (extract (UMAXV Mask), 0), 1),
+ // (extract Vec, (select_cc Find, -1, 0, Find, seteq)), Passthru
+ //
+ // becomes:
+ //
+ // select (setne Find, -1), Extract, Passthru
+ SDValue Cond = N->getOperand(0);
+ if (Cond.getOpcode() != ISD::AND)
+ return SDValue();
+
+ SDValue Reduced;
+ if (isOneConstant(Cond.getOperand(0)))
+ Reduced = Cond.getOperand(1);
+ else if (isOneConstant(Cond.getOperand(1)))
+ Reduced = Cond.getOperand(0);
+
+ SDValue Extract = N->getOperand(1);
+ if (!Reduced || Reduced.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+ !isNullConstant(Reduced.getOperand(1)) ||
+ Reduced.getOperand(0).getOpcode() != AArch64ISD::UMAXV ||
+ Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return SDValue();
+
+ SDValue SafeIdx = Extract.getOperand(1);
+ if (SafeIdx.getOpcode() != ISD::SELECT_CC ||
+ SafeIdx.getOperand(0).getOpcode() != AArch64ISD::FIND_LAST_ACTIVE ||
+ SafeIdx.getOperand(0) != SafeIdx.getOperand(3) ||
+ !isAllOnesConstant(SafeIdx.getOperand(1)) ||
+ !isNullConstant(SafeIdx.getOperand(2)) ||
+ cast<CondCodeSDNode>(SafeIdx.getOperand(4))->get() != ISD::SETEQ)
+ return SDValue();
+
+ SDValue Find = SafeIdx.getOperand(0);
+ SDValue FindMask = Find.getOperand(0);
+ while (FindMask.getOpcode() == ISD::TRUNCATE)
+ FindMask = FindMask.getOperand(0);
+ SDValue ReducedMask = Reduced.getOperand(0).getOperand(0);
+ while (ReducedMask.getOpcode() == ISD::BITCAST)
+ ReducedMask = ReducedMask.getOperand(0);
+ if (FindMask != ReducedMask)
+ return SDValue();
+
+ SDLoc DL(N);
+ SDValue AllInactive = DAG.getSignedConstant(-1, DL, Find.getValueType());
+ SDValue IsActive =
+ DAG.getSetCC(DL, Cond.getValueType(), Find, AllInactive, ISD::SETNE);
+ return DAG.getSelect(DL, N->getValueType(0), IsActive, Extract,
+ N->getOperand(2));
+}
+
/// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with
/// the compare-mask instructions rather than going via NZCV, even if LHS and
/// RHS are really scalar. This replaces any scalar setcc in the above pattern
@@ -30056,6 +30167,9 @@ static SDValue performSelectCombine(SDNode *N,
SDValue N0 = N->getOperand(0);
EVT ResVT = N->getValueType(0);
+ if (SDValue R = performFindLastActiveSelectCombine(N, DCI))
+ return R;
+
if (N0.getOpcode() != ISD::SETCC)
return SDValue();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 1b622e93480ec..fe18cc440d29c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -141,6 +141,9 @@ class AArch64TargetLowering : public TargetLowering {
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
+ /// Expand the sentinel-aware fixed mask node after DAG combining.
+ SDValue expandFixedLengthFindLastActive(SDValue Op, SelectionDAG &DAG) const;
+
/// This method returns a target specific FastISel object, or null if the
/// target does not support "fast" ISel.
FastISel *
@@ -670,6 +673,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerVECTOR_COMPRESS(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op, SelectionDAG &DAG) const;
+
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3b226e9a78105..17f10cf449a09 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1259,6 +1259,16 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 2,
[SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
+// res = lastp(pg, op)
+def AArch64LastP : SDNode<"AArch64ISD::LASTP", SDTypeProfile<1, 2,
+ [SDTCisInt<0>, SDTCVecEltisVT<1,i1>,
+ SDTCisSameAs<1,2>]>, []>;
+
+// Find the last active element in a fixed-length mask, returning -1 when the
+// mask is empty. This retains the original mask until after DAG combining.
+def AArch64FindLastActive : SDNode<"AArch64ISD::FIND_LAST_ACTIVE",
+ SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>, []>;
+
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
def AArch64ld2post : SDNode<"AArch64ISD::LD2post", SDTypeProfile<3, 2, []>,
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2987b1dc85a7c..df333828174c4 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -5071,6 +5071,18 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
defm FIRSTP_XPP : sve_int_pcount_pred<0b001, "firstp", int_aarch64_sve_firstp>;
defm LASTP_XPP : sve_int_pcount_pred<0b010, "lastp", int_aarch64_sve_lastp>;
+ def : Pat<(i64 (AArch64LastP nxv16i1:$Pg, nxv16i1:$Op1)),
+ (LASTP_XPP_B PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv8i1:$Pg, nxv8i1:$Op1)),
+ (LASTP_XPP_H PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv4i1:$Pg, nxv4i1:$Op1)),
+ (LASTP_XPP_S PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv2i1:$Pg, nxv2i1:$Op1)),
+ (LASTP_XPP_D PPR:$Pg, PPR:$Op1)>;
+
// SVE reverse within elements, zeroing predicate
defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit", AArch64rbit_mt>;
defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb", AArch64revb_mt>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 7eac36af4ab46..a82fa0152032e 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -49,6 +49,7 @@ void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
// Some additional checks not yet implemented by verifyTargetNode.
switch (N->getOpcode()) {
case AArch64ISD::CTTZ_ELTS:
+ case AArch64ISD::LASTP:
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
"Expected the general-predicate and mask to have matching types");
break;
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 2c7fd22622853..532a3a8831a5b 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED
; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE2P2-FIXED
define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; NEON-FIXED-LABEL: extract_last_i8:
@@ -45,6 +46,25 @@ define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i8:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: movi v2.2d, #0000000000000000
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl16
+; SVE2P2-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, z2.b
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfxil x9, x8, #0, #4
+; SVE2P2-FIXED-NEXT: ldrb w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <16 x i8> %mask, zeroinitializer
%res = call i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8> %data, <16 x i1> %notzero, i8 %passthru)
ret i8 %res
@@ -93,6 +113,25 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i16:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: ldrh w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16> %data, <8 x i1> %notzero, i16 %passthru)
ret i16 %res
@@ -141,6 +180,25 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.4h, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %data, <4 x i1> %notzero, i32 %passthru)
ret i32 %res
@@ -189,6 +247,25 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
; SVE-FIXED-NEXT: csel x0, x8, x0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i64:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.2s, v1.2d
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: ldr x8, [x9]
+; SVE2P2-FIXED-NEXT: csel x0, x8, x0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64> %data, <2 x i1> %notzero, i64 %passthru)
ret i64 %res
@@ -239,6 +316,25 @@ define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_half:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half> %data, <8 x i1> %notzero, half %passthru)
ret half %res
@@ -289,6 +385,25 @@ define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_bfloat:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat> %data, <8 x i1> %notzero, bfloat %passthru)
ret bfloat %res
@@ -337,6 +452,25 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; SVE-FIXED-NEXT: fcsel s0, s0, s2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_float:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.4h, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr s0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel s0, s0, s2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float> %data, <4 x i1> %notzero, float %passthru)
ret float %res
@@ -385,6 +519,25 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
; SVE-FIXED-NEXT: fcsel d0, d0, d2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_double:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.2s, v1.2d
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: ldr d0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel d0, d0, d2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double> %data, <2 x i1> %notzero, double %passthru)
ret double %res
@@ -546,36 +699,112 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; SVE-FIXED-NEXT: csinv w0, w8, wzr, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_active_v3i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: movi v1.2d, #0000000000000000
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: mov v1.h[0], w0
+; SVE2P2-FIXED-NEXT: mov v1.h[1], w1
+; SVE2P2-FIXED-NEXT: mov v1.h[2], w2
+; SVE2P2-FIXED-NEXT: shl v2.4h, v1.4h, #15
+; SVE2P2-FIXED-NEXT: orv h1, p0, z1.h
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z2.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csinv w0, w8, wzr, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%res = call i32 @llvm.experimental.vector.extract.last.active.v3i32(<3 x i32> %a, <3 x i1> %c, i32 -1)
ret i32 %res
}
define i8 @extract_last_active_split(<vscale x 32 x i8> %data, <vscale x 32 x i1> %mask, i8 %passthru) #0 {
-; CHECK-LABEL: extract_last_active_split:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: index z2.b, #0, #1
-; CHECK-NEXT: ptest p1, p1.b
-; CHECK-NEXT: rdvl x10, #1
-; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: lastb w8, p1, z2.b
-; CHECK-NEXT: lastb w9, p0, z2.b
-; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
-; CHECK-NEXT: add x8, x8, x10
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: csel x8, x8, x9, ne
-; CHECK-NEXT: sub x9, x10, #1
-; CHECK-NEXT: cmp x8, x9
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: ptest p0, p0.b
-; CHECK-NEXT: ldrb w8, [x9, x8]
-; CHECK-NEXT: csel w0, w8, w0, ne
-; CHECK-NEXT: addvl sp, sp, #2
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; NEON-FIXED-LABEL: extract_last_active_split:
+; NEON-FIXED: // %bb.0:
+; NEON-FIXED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NEON-FIXED-NEXT: addvl sp, sp, #-2
+; NEON-FIXED-NEXT: index z2.b, #0, #1
+; NEON-FIXED-NEXT: ptest p1, p1.b
+; NEON-FIXED-NEXT: rdvl x10, #1
+; NEON-FIXED-NEXT: str z1, [sp, #1, mul vl]
+; NEON-FIXED-NEXT: str z0, [sp]
+; NEON-FIXED-NEXT: lastb w8, p1, z2.b
+; NEON-FIXED-NEXT: lastb w9, p0, z2.b
+; NEON-FIXED-NEXT: sel p0.b, p0, p0.b, p1.b
+; NEON-FIXED-NEXT: add x8, x8, x10
+; NEON-FIXED-NEXT: rdvl x10, #2
+; NEON-FIXED-NEXT: csel x8, x8, x9, ne
+; NEON-FIXED-NEXT: sub x9, x10, #1
+; NEON-FIXED-NEXT: cmp x8, x9
+; NEON-FIXED-NEXT: csel x8, x8, x9, lo
+; NEON-FIXED-NEXT: mov x9, sp
+; NEON-FIXED-NEXT: ptest p0, p0.b
+; NEON-FIXED-NEXT: ldrb w8, [x9, x8]
+; NEON-FIXED-NEXT: csel w0, w8, w0, ne
+; NEON-FIXED-NEXT: addvl sp, sp, #2
+; NEON-FIXED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NEON-FIXED-NEXT: ret
+;
+; SVE-FIXED-LABEL: extract_last_active_split:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE-FIXED-NEXT: addvl sp, sp, #-2
+; SVE-FIXED-NEXT: index z2.b, #0, #1
+; SVE-FIXED-NEXT: ptest p1, p1.b
+; SVE-FIXED-NEXT: rdvl x10, #1
+; SVE-FIXED-NEXT: str z1, [sp, #1, mul vl]
+; SVE-FIXED-NEXT: str z0, [sp]
+; SVE-FIXED-NEXT: lastb w8, p1, z2.b
+; SVE-FIXED-NEXT: lastb w9, p0, z2.b
+; SVE-FIXED-NEXT: sel p0.b, p0, p0.b, p1.b
+; SVE-FIXED-NEXT: add x8, x8, x10
+; SVE-FIXED-NEXT: rdvl x10, #2
+; SVE-FIXED-NEXT: csel x8, x8, x9, ne
+; SVE-FIXED-NEXT: sub x9, x10, #1
+; SVE-FIXED-NEXT: cmp x8, x9
+; SVE-FIXED-NEXT: csel x8, x8, x9, lo
+; SVE-FIXED-NEXT: mov x9, sp
+; SVE-FIXED-NEXT: ptest p0, p0.b
+; SVE-FIXED-NEXT: ldrb w8, [x9, x8]
+; SVE-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE-FIXED-NEXT: addvl sp, sp, #2
+; SVE-FIXED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_active_split:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE2P2-FIXED-NEXT: addvl sp, sp, #-2
+; SVE2P2-FIXED-NEXT: index z2.b, #0, #1
+; SVE2P2-FIXED-NEXT: mov x10, #-1 // =0xffffffffffffffff
+; SVE2P2-FIXED-NEXT: ptest p1, p1.b
+; SVE2P2-FIXED-NEXT: incb x10, all, mul #2
+; SVE2P2-FIXED-NEXT: str z1, [sp, #1, mul vl]
+; SVE2P2-FIXED-NEXT: str z0, [sp]
+; SVE2P2-FIXED-NEXT: lastb w8, p1, z2.b
+; SVE2P2-FIXED-NEXT: lastb w9, p0, z2.b
+; SVE2P2-FIXED-NEXT: sel p0.b, p0, p0.b, p1.b
+; SVE2P2-FIXED-NEXT: incb x8
+; SVE2P2-FIXED-NEXT: csel x8, x8, x9, ne
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: cmp x8, x10
+; SVE2P2-FIXED-NEXT: csel x8, x8, x10, lo
+; SVE2P2-FIXED-NEXT: ptest p0, p0.b
+; SVE2P2-FIXED-NEXT: ldrb w8, [x9, x8]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: addvl sp, sp, #2
+; SVE2P2-FIXED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE2P2-FIXED-NEXT: ret
%res = call i8 @llvm.experimental.vector.extract.last.active.nxv32i8(<vscale x 32 x i8> %data, <vscale x 32 x i1> %mask, i8 %passthru)
ret i8 %res
}
More information about the llvm-commits
mailing list