[llvm] [AArch64] Lower fixed find_last_active to LASTP with SVE2.2 (PR #222534)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 00:59:35 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Vimal Patel (pvimal816a)
<details>
<summary>Changes</summary>
Add an AArch64 LASTP target node and use it when lowering fixed-length VECTOR_FIND_LAST_ACTIVE under SVE2.2/SME2.2.
Keep scalable predicate forms legal so existing LASTB/CLASTB combines can still handle extract-last-active cases.
---
Patch is 22.21 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222534.diff
6 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+32)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+2)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.td (+5)
- (modified) llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td (+12)
- (modified) llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp (+1)
- (modified) llvm/test/CodeGen/AArch64/vector-extract-last-active.ll (+278-25)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c51683b78cb54..5f48a3830b083 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2273,6 +2273,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ // Mark the fixed-length vector container types for custom lowering.
+ // These variants lower to lastp. The scalable variants lower to
+ // lastb/clastb.
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32, MVT::v1i64, MVT::v2i64})
+ setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Custom);
+
} else {
// Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
// which is not supported with for compact (with only +sve).
@@ -8257,6 +8265,28 @@ SDValue AArch64TargetLowering::LowerVECTOR_COMPRESS(SDValue Op,
Passthru);
}
+SDValue
+AArch64TargetLowering::LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT VT = Op.getValueType();
+ SDValue Mask = Op->getOperand(0);
+ EVT MaskVT = Mask.getValueType();
+ SDLoc DL(Op);
+ SDValue Pg;
+ if (MaskVT.isFixedLengthVector()) {
+ Pg = getPredicateForVector(DAG, DL, MaskVT);
+ Mask = convertFixedMaskToScalableVector(Mask, DAG);
+ } else {
+ Pg = getPredicateForVector(DAG, DL, MaskVT);
+ }
+ SDValue Lastp = DAG.getNode(AArch64ISD::LASTP, DL, VT, Pg, Mask);
+ EVT LastpVT = Lastp.getValueType();
+ SDValue LHS = DAG.getSignedConstant(-1, DL, LastpVT);
+ SDValue Zero = DAG.getSignedConstant(0, DL, LastpVT);
+ // Return 0 if there are no active elements.
+ return DAG.getSelectCC(DL, LHS, Lastp, Zero, Lastp, ISD::CondCode::SETEQ);
+}
+
SDValue AArch64TargetLowering::LowerSMULFIXSAT(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -8967,6 +8997,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVSCALE(Op, DAG);
case ISD::VECTOR_COMPRESS:
return LowerVECTOR_COMPRESS(Op, DAG);
+ case ISD::VECTOR_FIND_LAST_ACTIVE:
+ return LowerVECTOR_FIND_LAST_ACTIVE(Op, DAG);
case ISD::ANY_EXTEND:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 1b622e93480ec..ca640ee05323c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -670,6 +670,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerVECTOR_COMPRESS(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerVECTOR_FIND_LAST_ACTIVE(SDValue Op, SelectionDAG &DAG) const;
+
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3b226e9a78105..e6cd339ef85a4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1259,6 +1259,11 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 2,
[SDTCisInt<0>, SDTCVecEltisVT<1,i1>, SDTCisSameAs<1,2>]>, []>;
+// res = lastp(pg, op)
+def AArch64LastP : SDNode<"AArch64ISD::LASTP", SDTypeProfile<1, 2,
+ [SDTCisInt<0>, SDTCVecEltisVT<1,i1>,
+ SDTCisSameAs<1,2>]>, []>;
+
// NEON Load/Store with post-increment base updates.
// TODO: Complete SDTypeProfile constraints.
def AArch64ld2post : SDNode<"AArch64ISD::LD2post", SDTypeProfile<3, 2, []>,
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2987b1dc85a7c..df333828174c4 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -5071,6 +5071,18 @@ let Predicates = [HasSVE2p2_or_SME2p2] in {
defm FIRSTP_XPP : sve_int_pcount_pred<0b001, "firstp", int_aarch64_sve_firstp>;
defm LASTP_XPP : sve_int_pcount_pred<0b010, "lastp", int_aarch64_sve_lastp>;
+ def : Pat<(i64 (AArch64LastP nxv16i1:$Pg, nxv16i1:$Op1)),
+ (LASTP_XPP_B PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv8i1:$Pg, nxv8i1:$Op1)),
+ (LASTP_XPP_H PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv4i1:$Pg, nxv4i1:$Op1)),
+ (LASTP_XPP_S PPR:$Pg, PPR:$Op1)>;
+
+ def : Pat<(i64 (AArch64LastP nxv2i1:$Pg, nxv2i1:$Op1)),
+ (LASTP_XPP_D PPR:$Pg, PPR:$Op1)>;
+
// SVE reverse within elements, zeroing predicate
defm RBIT_ZPzZ : sve_int_perm_rev_rbit_z<"rbit", AArch64rbit_mt>;
defm REVB_ZPzZ : sve_int_perm_rev_revb_z<"revb", AArch64revb_mt>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 7eac36af4ab46..a82fa0152032e 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -49,6 +49,7 @@ void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
// Some additional checks not yet implemented by verifyTargetNode.
switch (N->getOpcode()) {
case AArch64ISD::CTTZ_ELTS:
+ case AArch64ISD::LASTP:
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
"Expected the general-predicate and mask to have matching types");
break;
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index 2c7fd22622853..83d5e143f6028 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED
; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE2P2-FIXED
define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; NEON-FIXED-LABEL: extract_last_i8:
@@ -45,6 +46,28 @@ define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i8:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl16
+; SVE2P2-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; SVE2P2-FIXED-NEXT: cmtst v1.16b, v1.16b, v1.16b
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: umaxv b1, v1.16b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfxil x9, x8, #0, #4
+; SVE2P2-FIXED-NEXT: ldrb w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <16 x i8> %mask, zeroinitializer
%res = call i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8> %data, <16 x i1> %notzero, i8 %passthru)
ret i8 %res
@@ -93,6 +116,28 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i16:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v2.8b, v1.8h
+; SVE2P2-FIXED-NEXT: umaxv h1, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z2.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: ldrh w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16> %data, <8 x i1> %notzero, i16 %passthru)
ret i16 %res
@@ -141,6 +186,28 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; SVE-FIXED-NEXT: csel w0, w8, w0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v2.4h, v1.4s
+; SVE2P2-FIXED-NEXT: umaxv s1, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z2.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csel w0, w8, w0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %data, <4 x i1> %notzero, i32 %passthru)
ret i32 %res
@@ -189,6 +256,28 @@ define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {
; SVE-FIXED-NEXT: csel x0, x8, x0, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_i64:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v2.2s, v1.2d
+; SVE2P2-FIXED-NEXT: umaxv s1, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: ldr x8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csel x0, x8, x0, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64> %data, <2 x i1> %notzero, i64 %passthru)
ret i64 %res
@@ -239,6 +328,28 @@ define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_half:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v3.8b, v1.8h
+; SVE2P2-FIXED-NEXT: umaxv h1, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z3.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: fmov w8, s1
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: tst w8, #0x1
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half> %data, <8 x i1> %notzero, half %passthru)
ret half %res
@@ -289,6 +400,28 @@ define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %
; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_bfloat:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
+; SVE2P2-FIXED-NEXT: ptrue p0.b, vl8
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v3.8b, v1.8h
+; SVE2P2-FIXED-NEXT: umaxv h1, v1.8h
+; SVE2P2-FIXED-NEXT: cmpne p1.b, p0/z, z3.b, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.b
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #1, #3
+; SVE2P2-FIXED-NEXT: fmov w8, s1
+; SVE2P2-FIXED-NEXT: ldr h0, [x9]
+; SVE2P2-FIXED-NEXT: tst w8, #0x1
+; SVE2P2-FIXED-NEXT: fcsel h0, h0, h2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <8 x i16> %mask, zeroinitializer
%res = call bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat> %data, <8 x i1> %notzero, bfloat %passthru)
ret bfloat %res
@@ -337,6 +470,28 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; SVE-FIXED-NEXT: fcsel s0, s0, s2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_float:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v3.4h, v1.4s
+; SVE2P2-FIXED-NEXT: umaxv s1, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z3.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: fmov w8, s1
+; SVE2P2-FIXED-NEXT: ldr s0, [x9]
+; SVE2P2-FIXED-NEXT: tst w8, #0x1
+; SVE2P2-FIXED-NEXT: fcsel s0, s0, s2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <4 x i32> %mask, zeroinitializer
%res = call float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float> %data, <4 x i1> %notzero, float %passthru)
ret float %res
@@ -385,6 +540,28 @@ define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %
; SVE-FIXED-NEXT: fcsel d0, d0, d2, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_double:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; SVE2P2-FIXED-NEXT: ptrue p0.s, vl2
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: xtn v3.2s, v1.2d
+; SVE2P2-FIXED-NEXT: umaxv s1, v1.4s
+; SVE2P2-FIXED-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.s
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #3, #1
+; SVE2P2-FIXED-NEXT: fmov w8, s1
+; SVE2P2-FIXED-NEXT: ldr d0, [x9]
+; SVE2P2-FIXED-NEXT: tst w8, #0x1
+; SVE2P2-FIXED-NEXT: fcsel d0, d0, d2, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%notzero = icmp ne <2 x i64> %mask, zeroinitializer
%res = call double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double> %data, <2 x i1> %notzero, double %passthru)
ret double %res
@@ -546,36 +723,112 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; SVE-FIXED-NEXT: csinv w0, w8, wzr, ne
; SVE-FIXED-NEXT: add sp, sp, #16
; SVE-FIXED-NEXT: ret
+;
+; SVE2P2-FIXED-LABEL: extract_last_active_v3i32:
+; SVE2P2-FIXED: // %bb.0:
+; SVE2P2-FIXED-NEXT: sub sp, sp, #16
+; SVE2P2-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE2P2-FIXED-NEXT: movi v1.2d, #0000000000000000
+; SVE2P2-FIXED-NEXT: ptrue p0.h, vl4
+; SVE2P2-FIXED-NEXT: mov x9, sp
+; SVE2P2-FIXED-NEXT: str q0, [sp]
+; SVE2P2-FIXED-NEXT: mov v1.h[0], w0
+; SVE2P2-FIXED-NEXT: mov v1.h[1], w1
+; SVE2P2-FIXED-NEXT: mov v1.h[2], w2
+; SVE2P2-FIXED-NEXT: shl v2.4h, v1.4h, #15
+; SVE2P2-FIXED-NEXT: orv h1, p0, z1.h
+; SVE2P2-FIXED-NEXT: cmpne p1.h, p0/z, z2.h, #0
+; SVE2P2-FIXED-NEXT: lastp x8, p0, p1.h
+; SVE2P2-FIXED-NEXT: cmn x8, #1
+; SVE2P2-FIXED-NEXT: csel x8, xzr, x8, eq
+; SVE2P2-FIXED-NEXT: bfi x9, x8, #2, #2
+; SVE2P2-FIXED-NEXT: ldr w8, [x9]
+; SVE2P2-FIXED-NEXT: fmov w9, s1
+; SVE2P2-FIXED-NEXT: tst w9, #0x1
+; SVE2P2-FIXED-NEXT: csinv w0, w8, wzr, ne
+; SVE2P2-FIXED-NEXT: add sp, sp, #16
+; SVE2P2-FIXED-NEXT: ret
%res = call i32 @llvm.experimental.vector.extract.last.active.v3i32(<3 x i32> %a, <3 x i1> %c, i32 -1)
ret i32 %res
}
define i8 @extract_last_active_split(<vscale x 32 x i8> %data, <vscale x 32 x i1> %mask, i8 %passthru) #0 {
-; CHECK-LABEL: extract_last_active_split:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: index z2.b, #0, #1
-; CHECK-NEXT: ptest p1, p1.b
-; CHECK-NEXT: rdvl x10, #1
-; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: lastb w8, p1, z2.b
-; CHECK-NEXT: lastb w9, p0, z2.b
-; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
-; CHECK-NEXT: add x8, x8, x10
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: csel x8, x8, x9, ne
-; CHECK-NEXT: sub x9, x10, #1
-; CHECK-NEXT: cmp x8, x9
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: ptest p0, p0.b
-; CHECK-NEXT: ldrb w8, [x9, x8]
-; CHECK-NEXT: csel w0, w8, w0, ne
-; CHECK-NEXT: addvl sp, sp, #2
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; NEON-FIXED-LABEL: extract_last_active_split:
+; NEON-FIXED: // %bb.0:
+; NEON-FIXED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NEON-FIXED-NEXT: addvl sp, sp, #-2
+; NEON-FIXED-NEXT: index z2.b, #0, #1
+; NEON-FIXED-NEXT: ptest p1, p1.b
+; NEON-FIXED-NEXT: rdvl x10, #1
+; NEON-FIXED-NEXT: str z1, [sp, #1, mul vl]
+; NEON-FIXED-NEXT: str z0, [sp]
+; NEON-FIXED-NEXT: lastb w8, p1, z2.b
+; NEON-FIXED-NEXT: lastb w9, p0, z2.b
+; NEON-FIXED-NEXT: sel p0.b, p0, p0.b, p1.b
+; NEON-FIXED-NEXT: add x8, x8, x10
+; NEON-FIXED-NEXT: rdvl x10, #2
+; NEON-FIXED-NEXT: csel x8, x8, x9, ne
+; NEON-FIXED-NEXT: sub x9, x10, #1
+; NEON-FIXED-NEXT: cmp x8, x9
+; NEON-FIXED-NEXT: csel x8, x8, x9, lo
+; NEON-FIXED-NEXT: mov x9, sp
+; NEON-FIXED-NEXT: ptest p0, p0.b
+; NEON-FIXED-NEXT: ldrb w8, [x9, x8]
+; NEON-FIXED-NEXT: csel w0, w8, w0, ne
+; NEON-FIXED-NEXT: addvl sp, sp, #2
+; NEON-FIXED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NEON-FIXED-NEXT: ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222534
More information about the llvm-commits
mailing list