[llvm] [AArch64] Lower fixed find_last_active to LASTP with SVE2.2 (PR #222534)
Vimal Patel via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 04:20:43 PDT 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/222534
>From b1db7bc7c0984100a19a77b7f9f6fe530dd14f39 Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Tue, 8 Sep 2026 12:55:19 +0000
Subject: [PATCH] [AArch64] Lower fixed find_last_active to LASTP with SVE2.2
Lower ISD::VECTOR_FIND_LAST_ACTIVE to LASTP when SVE2.2/SME2.2 is
available. But, the existing extract(find_last_active) combiner will
still get a chance to run if applicable and in that case it generates
LASTB instead, even when SVE2.2/SME2.2 is available. The LASTP
instruction returns -1, but to avoid undefined behaviour -1 will be
replaced with 0.
Additionally a new combiner for select node is added. In the context of
`llvm.experimental.vector.extract.last.active` this select node selects
between the passthrough value and the value extracted from the data
vector. Computing the condition for this select could just rely on the
result of LASTP under SVE2.2/SME2.2. So, the combiner added will rewrite
the condition to rely on result of LASTP node instead of using UMAXV for
it.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 +++
.../Target/AArch64/AArch64ISelLowering.cpp | 100 ++++++-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 10 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 11 +
.../AArch64/AArch64SelectionDAGInfo.cpp | 1 +
.../AArch64/vector-extract-last-active.ll | 266 ++++++++++++++++--
7 files changed, 395 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index feb8f465837940..8b2d74c39fdf13 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -8300,6 +8300,38 @@ void AArch64DAGToDAGISel::PreprocessISelDAG() {
SDValue Result;
switch (N.getOpcode()) {
+ case AArch64ISD::FIND_LAST_ACTIVE: {
+ const auto *TLI = static_cast<const AArch64TargetLowering *>(
+ Subtarget->getTargetLowering());
+ Result = TLI->expandFindLastActive(SDValue(&N, 0), *CurDAG);
+ assert(Result.getOpcode() == AArch64ISD::LASTP &&
+ "FIND_LAST_ACTIVE must expand to LASTP");
+
+ // LASTP returns -1 for an empty mask. Convert that into the target
+ // node's two results here: a safe index for extraction and a separate
+ // validity value for empty-mask checks.
+ SDLoc DL(&N);
+ SDValue Zero = CurDAG->getSignedConstant(0, DL, Result.getValueType());
+ SDValue One = CurDAG->getSignedConstant(1, DL, Result.getValueType());
+ SDValue Add = CurDAG->getNode(
+ AArch64ISD::ADDS, DL,
+ CurDAG->getVTList(Result.getValueType(), MVT::i32), Result, One);
+ SDValue CC = CurDAG->getConstant(AArch64CC::EQ, DL, MVT::i32);
+ SDValue SafeIndex =
+ CurDAG->getNode(AArch64ISD::CSEL, DL, Result.getValueType(), Zero,
+ Result, CC, Add.getValue(1));
+
+ SDValue Zero32 = CurDAG->getSignedConstant(0, DL, MVT::i32);
+ SDValue One32 = CurDAG->getSignedConstant(1, DL, MVT::i32);
+ SDValue Valid = CurDAG->getNode(AArch64ISD::CSEL, DL, MVT::i32, Zero32,
+ One32, CC, Add.getValue(1));
+
+ SDValue From[] = {SDValue(&N, 0), SDValue(&N, 1)};
+ SDValue To[] = {SafeIndex, Valid};
+ CurDAG->ReplaceAllUsesOfValuesWith(From, To, 2);
+ MadeChange = true;
+ continue;
+ }
case ISD::SCALAR_TO_VECTOR: {
EVT ScalarTy = N.getValueType(0).getVectorElementType();
if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d90030ee5519e7..7e57ec8880b720 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2289,6 +2289,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ // Mark the scalable predicate and fixed-length vector container types
+ // for custom lowering.
+ // Extracts can still combine to lastb/clastb, otherwise the target node
+ // lowers to lastp.
+ for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1,
+ MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32, MVT::v1i64, MVT::v2i64})
+ setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Custom);
+
} else {
// Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
// which is not supported with for compact (with only +sve).
@@ -8274,6 +8284,26 @@ SDValue AArch64TargetLowering::LowerVECTOR_COMPRESS(SDValue Op,
Passthru);
}
+SDValue
+AArch64TargetLowering::LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
+ return DAG.getNode(AArch64ISD::FIND_LAST_ACTIVE, SDLoc(Op), VTs,
+ Op.getOperand(0));
+}
+
+SDValue AArch64TargetLowering::expandFindLastActive(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Op.getOpcode() == AArch64ISD::FIND_LAST_ACTIVE);
+ SDLoc DL(Op);
+ SDValue Mask = Op.getOperand(0);
+ EVT MaskVT = Mask.getValueType();
+ SDValue Pg = getPredicateForVector(DAG, DL, MaskVT);
+ if (!MaskVT.isScalableVector())
+ Mask = convertFixedMaskToScalableVector(Mask, DAG);
+ return DAG.getNode(AArch64ISD::LASTP, DL, Op.getValueType(), Pg, Mask);
+}
+
SDValue AArch64TargetLowering::LowerSMULFIXSAT(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -8981,6 +9011,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVSCALE(Op, DAG);
case ISD::VECTOR_COMPRESS:
return LowerVECTOR_COMPRESS(Op, DAG);
+ case ISD::VECTOR_FIND_LAST_ACTIVE:
+ return LowerVECTOR_FIND_LAST_ACTIVE(Op, DAG);
case ISD::ANY_EXTEND:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
@@ -22544,7 +22576,11 @@ performExtractLastActiveCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SDValue Vec = N->getOperand(0);
SDValue Idx = N->getOperand(1);
- if (DCI.isBeforeLegalize() || Idx.getOpcode() != ISD::VECTOR_FIND_LAST_ACTIVE)
+ // A custom lowering is used which introduces target specific node when
+ // sve2.2/sme2.2 is available.
+ if (DCI.isBeforeLegalize() ||
+ (Idx.getOpcode() != ISD::VECTOR_FIND_LAST_ACTIVE &&
+ Idx.getOpcode() != AArch64ISD::FIND_LAST_ACTIVE))
return SDValue();
// Only legal for 8, 16, 32, and 64 bit element types.
@@ -22555,8 +22591,9 @@ performExtractLastActiveCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return SDValue();
SDValue Mask = Idx.getOperand(0);
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- if (!TLI.isOperationLegal(ISD::VECTOR_FIND_LAST_ACTIVE, Mask.getValueType()))
+ if (!is_contained(
+ ArrayRef({MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}),
+ Mask.getSimpleValueType().SimpleTy))
return SDValue();
return DAG.getNode(AArch64ISD::LASTB, SDLoc(N), N->getValueType(0), Mask,
@@ -30222,6 +30259,60 @@ static SDValue performVSelectCombine(SDNode *N,
IfTrue, IfFalse);
}
+static SDValue
+performFindLastActiveSelectCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ if (DCI.isBeforeLegalize())
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ // Replace the reduction used to detect an empty mask with the validity result
+ // returned by FIND_LAST_ACTIVE.
+ //
+ // select (and (extract (UMAXV Mask), 0), 1),
+ // (extract Vec, Find), Passthru
+ //
+ // becomes:
+ //
+ // select (Find.valid), Extract, Passthru
+ SDValue Cond = N->getOperand(0);
+ if (Cond.getOpcode() != ISD::AND)
+ return SDValue();
+
+ SDValue Reduced;
+ if (isOneConstant(Cond.getOperand(0)))
+ Reduced = Cond.getOperand(1);
+ else if (isOneConstant(Cond.getOperand(1)))
+ Reduced = Cond.getOperand(0);
+
+ SDValue Extract = N->getOperand(1);
+ if (!Reduced || Reduced.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+ !isNullConstant(Reduced.getOperand(1)) ||
+ Reduced.getOperand(0).getOpcode() != AArch64ISD::UMAXV ||
+ Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return SDValue();
+
+ SDValue Find = Extract.getOperand(1);
+ if (Find.getOpcode() != AArch64ISD::FIND_LAST_ACTIVE)
+ return SDValue();
+
+ SDValue FindMask = Find.getOperand(0);
+ while (FindMask.getOpcode() == ISD::TRUNCATE)
+ FindMask = FindMask.getOperand(0);
+ SDValue ReducedMask = Reduced.getOperand(0).getOperand(0);
+ while (ReducedMask.getOpcode() == ISD::BITCAST)
+ ReducedMask = ReducedMask.getOperand(0);
+ if (FindMask != ReducedMask)
+ return SDValue();
+
+ SDLoc DL(N);
+ SDValue IsActive = Find.getValue(1);
+ if (IsActive.getValueType() != Cond.getValueType())
+ IsActive = DAG.getZExtOrTrunc(IsActive, DL, Cond.getValueType());
+ return DAG.getSelect(DL, N->getValueType(0), IsActive, Extract,
+ N->getOperand(2));
+}
+
/// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with
/// the compare-mask instructions rather than going via NZCV, even if LHS and
/// RHS are really scalar. This replaces any scalar setcc in the above pattern
@@ -30232,6 +30323,9 @@ static SDValue performSelectCombine(SDNode *N,
SDValue N0 = N->getOperand(0);
EVT ResVT = N->getValueType(0);
+ if (SDValue R = performFindLastActiveSelectCombine(N, DCI))
+ return R;
+
if (N0.getOpcode() != ISD::SETCC)
return SDValue();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b68e06dede580a..f24763e90f93bf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -141,6 +141,9 @@ class AArch64TargetLowering : public TargetLowering {
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
+ /// Expand the sentinel-aware find_last_active node after DAG combining.
+ SDValue expandFindLastActive(SDValue Op, SelectionDAG &DAG) const;
+
/// This method returns a target specific FastISel object, or null if the
/// target does not support "fast" ISel.
FastISel *
@@ -672,6 +675,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerVECTOR_COMPRESS(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op, SelectionDAG &DAG) const;
+
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3c2fd32f83d754..5eaa544c4d6765 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1262,6 +1262,16 @@ def AArch64CttzEltsZeroPoison
: SDNode<"AArch64ISD::CTTZ_ELTS_ZERO_POISON", SDTypeProfile<1, 2,
[SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
+// res = lastp(pg, op)
+def AArch64LastP : SDNode<"AArch64ISD::LASTP", SDTypeProfile<1, 2,
+ [SDTCisInt<0>, SDTCVecEltisVT<1,i1>,
+ SDTCisSameAs<1,2>]>, []>;
+
+// Find the last active element in a mask. Result 0 is a safe index and result
+// 1 is nonzero when the mask contains an active element.
+def AArch64FindLastActive : SDNode<"AArch64ISD::FIND_LAST_ACTIVE",
+ SDTypeProfile<2, 1, [SDTCisInt<0>, SDTCisVT<1, i32>, SDTCisVec<2>]>, []>;
+
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
def AArch64ld2post : SDNode<"AArch64ISD::LD2post", SDTypeProfile<3, 2, []>,
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7d50afe3cbaa96..c37a38437eec73 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -5008,6 +5008,12 @@ defm TBLQ_ZZZ : sve2p1_tblq<"tblq", int_aarch64_sve_tblq>;
//===----------------------------------------------------------------------===//
// SME2.2 or SVE2.2 instructions
//===----------------------------------------------------------------------===//
+
+multiclass aarch64_lastp_pat<ValueType vtd, Instruction inst> {
+ def : Pat<(i64 (AArch64LastP vtd:$Pg, vtd:$Op1)),
+ (inst PPR:$Pg, PPR:$Op1)>;
+}
+
let Predicates = [HasSVE2p2_or_SME2p2] in {
// SVE Floating-point convert precision, zeroing predicate
defm FCVT_ZPzZ : sve_fp_z2op_p_zd_b_0<"fcvt", "int_aarch64_sve_fcvt">;
@@ -5102,6 +5108,11 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
def : Pat<(i64 (AArch64CttzEltsZeroPoison nxv2i1:$Pg, nxv2i1:$Op1)),
(FIRSTP_XPP_D PPR:$Pg, PPR:$Op1)>;
+ defm : aarch64_lastp_pat<nxv16i1, LASTP_XPP_B>;
+ defm : aarch64_lastp_pat<nxv8i1, LASTP_XPP_H>;
+ defm : aarch64_lastp_pat<nxv4i1, LASTP_XPP_S>;
+ defm : aarch64_lastp_pat<nxv2i1, LASTP_XPP_D>;
+
// SVE reverse within elements, zeroing predicate
defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit", AArch64rbit_mt>;
defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb", AArch64revb_mt>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 0e9f1a00a3e364..bd3ca33c06713a 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -50,6 +50,7 @@ void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
switch (N->getOpcode()) {
case AArch64ISD::CTTZ_ELTS:
case AArch64ISD::CTTZ_ELTS_ZERO_POISON:
+ case AArch64ISD::LASTP:
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
"Expected the general-predicate and mask to have matching types");
break;
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 2c7fd226228532..5448199a4d8fec 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED
-; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NO-SVE2P2,NEON-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NO-SVE2P2,SVE-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE2P2-FIXED
define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; NEON-FIXED-LABEL: extract_last_i8:
@@ -45,6 +46,25 @@ define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i8:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: movi v2.2d, #0000000000000000
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl16
+; SVE2P2-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, z2.b
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfxil x9, x8, #0, #4
+; SVE2P2-FIXED-NEXT: ldrb w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <16 x i8> %mask, zeroinitializer
%res = call i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8> %data, <16 x i1> %notzero, i8 %passthru)
ret i8 %res
@@ -93,6 +113,25 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i16:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: ldrh w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16> %data, <8 x i1> %notzero, i16 %passthru)
ret i16 %res
@@ -141,6 +180,25 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.4h, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %data, <4 x i1> %notzero, i32 %passthru)
ret i32 %res
@@ -189,6 +247,25 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
; SVE-FIXED-NEXT: csel x0, x8, x0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i64:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.2s, v1.2d
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: ldr x8, [x9]
+; SVE2P2-FIXED-NEXT: csel x0, x8, x0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64> %data, <2 x i1> %notzero, i64 %passthru)
ret i64 %res
@@ -239,6 +316,27 @@ define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_half:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: cset w8, ne
+; SVE2P2-FIXED-NEXT: cmp w8, #0
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half> %data, <8 x i1> %notzero, half %passthru)
ret half %res
@@ -289,6 +387,27 @@ define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_bfloat:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.8b, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: cset w8, ne
+; SVE2P2-FIXED-NEXT: cmp w8, #0
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat> %data, <8 x i1> %notzero, bfloat %passthru)
ret bfloat %res
@@ -337,6 +456,25 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; SVE-FIXED-NEXT: fcsel s0, s0, s2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_float:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.4h, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr s0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel s0, s0, s2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float> %data, <4 x i1> %notzero, float %passthru)
ret float %res
@@ -385,6 +523,25 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
; SVE-FIXED-NEXT: fcsel d0, d0, d2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_double:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v1.2s, v1.2d
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: ldr d0, [x9]
+; SVE2P2-FIXED-NEXT: fcsel d0, d0, d2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double> %data, <2 x i1> %notzero, double %passthru)
ret double %res
@@ -546,36 +703,91 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; SVE-FIXED-NEXT: csinv w0, w8, wzr, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_active_v3i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: movi v1.2d, #0000000000000000
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: mov v1.h[0], w0
+; SVE2P2-FIXED-NEXT: mov v1.h[1], w1
+; SVE2P2-FIXED-NEXT: mov v1.h[2], w2
+; SVE2P2-FIXED-NEXT: shl v2.4h, v1.4h, #15
+; SVE2P2-FIXED-NEXT: orv h1, p0, z1.h
+; SVE2P2-FIXED-NEXT: cmlt v2.4h, v2.4h, #0
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z2.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csinv w0, w8, wzr, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%res = call i32 @llvm.experimental.vector.extract.last.active.v3i32(<3 x i32> %a, <3 x i1> %c, i32 -1)
ret i32 %res
}
define i8 @extract_last_active_split(<vscale x 32 x i8> %data, <vscale x 32 x i1> %mask, i8 %passthru) #0 {
-; CHECK-LABEL: extract_last_active_split:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: index z2.b, #0, #1
-; CHECK-NEXT: ptest p1, p1.b
-; CHECK-NEXT: rdvl x10, #1
-; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: lastb w8, p1, z2.b
-; CHECK-NEXT: lastb w9, p0, z2.b
-; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
-; CHECK-NEXT: add x8, x8, x10
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: csel x8, x8, x9, ne
-; CHECK-NEXT: sub x9, x10, #1
-; CHECK-NEXT: cmp x8, x9
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: ptest p0, p0.b
-; CHECK-NEXT: ldrb w8, [x9, x8]
-; CHECK-NEXT: csel w0, w8, w0, ne
-; CHECK-NEXT: addvl sp, sp, #2
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; NO-SVE2P2-LABEL: extract_last_active_split:
+; NO-SVE2P2: // %bb.0:
+; NO-SVE2P2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO-SVE2P2-NEXT: addvl sp, sp, #-2
+; NO-SVE2P2-NEXT: index z2.b, #0, #1
+; NO-SVE2P2-NEXT: ptest p1, p1.b
+; NO-SVE2P2-NEXT: rdvl x10, #1
+; NO-SVE2P2-NEXT: str z1, [sp, #1, mul vl]
+; NO-SVE2P2-NEXT: str z0, [sp]
+; NO-SVE2P2-NEXT: lastb w8, p1, z2.b
+; NO-SVE2P2-NEXT: lastb w9, p0, z2.b
+; NO-SVE2P2-NEXT: sel p0.b, p0, p0.b, p1.b
+; NO-SVE2P2-NEXT: add x8, x8, x10
+; NO-SVE2P2-NEXT: rdvl x10, #2
+; NO-SVE2P2-NEXT: csel x8, x8, x9, ne
+; NO-SVE2P2-NEXT: sub x9, x10, #1
+; NO-SVE2P2-NEXT: cmp x8, x9
+; NO-SVE2P2-NEXT: csel x8, x8, x9, lo
+; NO-SVE2P2-NEXT: mov x9, sp
+; NO-SVE2P2-NEXT: ptest p0, p0.b
+; NO-SVE2P2-NEXT: ldrb w8, [x9, x8]
+; NO-SVE2P2-NEXT: csel w0, w8, w0, ne
+; NO-SVE2P2-NEXT: addvl sp, sp, #2
+; NO-SVE2P2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO-SVE2P2-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_active_split:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE2P2-FIXED-NEXT: addvl sp, sp, #-2
+; SVE2P2-FIXED-NEXT: ptrue p2.b
+; SVE2P2-FIXED-NEXT: mov x10, #-1 // =0xffffffffffffffff
+; SVE2P2-FIXED-NEXT: str z1, [sp, #1, mul vl]
+; SVE2P2-FIXED-NEXT: incb x10, all, mul #2
+; SVE2P2-FIXED-NEXT: str z0, [sp]
+; SVE2P2-FIXED-NEXT: lastp x8, p2, p0.b
+; SVE2P2-FIXED-NEXT: lastp x9, p2, p1.b
+; SVE2P2-FIXED-NEXT: sel p0.b, p0, p0.b, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: cmn x9, #1
+; SVE2P2-FIXED-NEXT: csel x9, xzr, x9, eq
+; SVE2P2-FIXED-NEXT: ptest p1, p1.b
+; SVE2P2-FIXED-NEXT: incb x9
+; SVE2P2-FIXED-NEXT: csel x8, x9, x8, ne
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: cmp x8, x10
+; SVE2P2-FIXED-NEXT: csel x8, x8, x10, lo
+; SVE2P2-FIXED-NEXT: ptest p0, p0.b
+; SVE2P2-FIXED-NEXT: ldrb w8, [x9, x8]
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: addvl sp, sp, #2
+; SVE2P2-FIXED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE2P2-FIXED-NEXT: ret
%res = call i8 @llvm.experimental.vector.extract.last.active.nxv32i8(<vscale x 32 x i8> %data, <vscale x 32 x i1> %mask, i8 %passthru)
ret i8 %res
}
More information about the llvm-commits
mailing list