[llvm] [AArch64] Combine getActiveLaneMask with vector_extract (PR #81139)
Momchil Velikov via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 1 03:51:56 PST 2024
https://github.com/momchil-velikov updated https://github.com/llvm/llvm-project/pull/81139
>From 28cdbbef1258394c491ae884b759dc99df9edf84 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov at arm.com>
Date: Fri, 22 Dec 2023 11:30:22 +0000
Subject: [PATCH] [AArch64] Combine getActiveLaneMask with vector_extract
... into a `whilelo` instruction with a pair of predicate registers.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 122 +++--
llvm/test/CodeGen/AArch64/active_lane_mask.ll | 1 +
.../AArch64/get-active-lane-mask-extract.ll | 427 ++++++++++++++++++
3 files changed, 515 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b1677df56e1bea..b27624a8357726 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1815,8 +1815,8 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
EVT OpVT) const {
- // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
- if (!Subtarget->hasSVE())
+ // Only SVE/SME has a 1:1 mapping from intrinsic -> instruction (whilelo).
+ if (!Subtarget->hasSVEorSME())
return true;
// We can only support legal predicate result types. We can use the SVE
@@ -20054,47 +20054,99 @@ static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc,
return SDValue();
}
-static SDValue performIntrinsicCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI,
- const AArch64Subtarget *Subtarget) {
+static SDValue tryCombineGetActiveLaneMask(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const AArch64Subtarget *Subtarget) {
SelectionDAG &DAG = DCI.DAG;
- unsigned IID = getIntrinsicID(N);
- switch (IID) {
- default:
- break;
- case Intrinsic::get_active_lane_mask: {
- SDValue Res = SDValue();
- EVT VT = N->getValueType(0);
- if (VT.isFixedLengthVector()) {
- // We can use the SVE whilelo instruction to lower this intrinsic by
- // creating the appropriate sequence of scalable vector operations and
- // then extracting a fixed-width subvector from the scalable vector.
+ EVT VT = N->getValueType(0);
+ if (VT.isFixedLengthVector()) {
+ // We can use the SVE whilelo instruction to lower this intrinsic by
+ // creating the appropriate sequence of scalable vector operations and
+ // then extracting a fixed-width subvector from the scalable vector.
+ SDLoc DL(N);
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, DL, MVT::i64);
- SDLoc DL(N);
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, DL, MVT::i64);
+ EVT WhileVT =
+ EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+ ElementCount::getScalable(VT.getVectorNumElements()));
- EVT WhileVT = EVT::getVectorVT(
- *DAG.getContext(), MVT::i1,
- ElementCount::getScalable(VT.getVectorNumElements()));
+ // Get promoted scalable vector VT, i.e. promote nxv4i1 -> nxv4i32.
+ EVT PromVT = getPromotedVTForPredicate(WhileVT);
- // Get promoted scalable vector VT, i.e. promote nxv4i1 -> nxv4i32.
- EVT PromVT = getPromotedVTForPredicate(WhileVT);
+ // Get the fixed-width equivalent of PromVT for extraction.
+ EVT ExtVT =
+ EVT::getVectorVT(*DAG.getContext(), PromVT.getVectorElementType(),
+ VT.getVectorElementCount());
- // Get the fixed-width equivalent of PromVT for extraction.
- EVT ExtVT =
- EVT::getVectorVT(*DAG.getContext(), PromVT.getVectorElementType(),
- VT.getVectorElementCount());
+ SDValue Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WhileVT, ID,
+ N->getOperand(1), N->getOperand(2));
+ Res = DAG.getNode(ISD::SIGN_EXTEND, DL, PromVT, Res);
+ Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtVT, Res,
+ DAG.getConstant(0, DL, MVT::i64));
+ Res = DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
- Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WhileVT, ID,
- N->getOperand(1), N->getOperand(2));
- Res = DAG.getNode(ISD::SIGN_EXTEND, DL, PromVT, Res);
- Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtVT, Res,
- DAG.getConstant(0, DL, MVT::i64));
- Res = DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
- }
return Res;
}
+
+ const uint64_t HalfSize = VT.getVectorMinNumElements() / 2;
+ if (HalfSize < 2)
+ return SDValue();
+
+ if (!Subtarget->hasSVE2p1() && !Subtarget->hasSME2())
+ return SDValue();
+
+ if (!N->hasNUsesOfValue(2, 0))
+ return SDValue();
+
+ auto It = N->use_begin();
+ SDNode *Lo = *It++;
+ SDNode *Hi = *It;
+
+ uint64_t OffLo, OffHi;
+ if (Lo->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+ !isIntImmediate(Lo->getOperand(1).getNode(), OffLo) ||
+ Hi->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+ !isIntImmediate(Hi->getOperand(1).getNode(), OffHi))
+ return SDValue();
+
+ if (OffLo > OffHi) {
+ std::swap(Lo, Hi);
+ std::swap(OffLo, OffHi);
+ }
+
+ if (OffLo != 0 || OffHi != HalfSize)
+ return SDValue();
+
+ SDLoc DL(N);
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo_x2, DL, MVT::i64);
+ SDValue Idx = N->getOperand(1);
+ SDValue TC = N->getOperand(2);
+ if (Idx.getValueType() != MVT::i64) {
+ Idx = DAG.getZExtOrTrunc(Idx, DL, MVT::i64);
+ TC = DAG.getZExtOrTrunc(TC, DL, MVT::i64);
+ }
+ auto R =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
+ {Lo->getValueType(0), Hi->getValueType(0)}, {ID, Idx, TC});
+
+ DCI.CombineTo(Lo, R.getValue(0));
+ DCI.CombineTo(Hi, R.getValue(1));
+
+ return SDValue(N, 0);
+}
+
+static SDValue performIntrinsicCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const AArch64Subtarget *Subtarget) {
+ SelectionDAG &DAG = DCI.DAG;
+ unsigned IID = getIntrinsicID(N);
+ switch (IID) {
+ default:
+ break;
+ case Intrinsic::get_active_lane_mask:
+ return tryCombineGetActiveLaneMask(N, DCI, Subtarget);
case Intrinsic::aarch64_neon_vcvtfxs2fp:
case Intrinsic::aarch64_neon_vcvtfxu2fp:
return tryCombineFixedPointConvert(N, DCI, DAG);
diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll
index a65c5d66677946..6a509b5f3afcae 100644
--- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll
+++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme < %s | FileCheck %s
; == Scalable ==
diff --git a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
new file mode 100644
index 00000000000000..232cb56ed6f20b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
@@ -0,0 +1,427 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mattr=+sve < %s | FileCheck %s -check-prefix CHECK-SVE
+; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s -check-prefix CHECK-SVE2p1
+; RUN: llc -mattr=+sme2 < %s | FileCheck %s -check-prefix CHECK-SME2
+target triple = "aarch64-linux"
+
+; Test combining of getActiveLaneMask with a pair of extract_vector operations.
+
+define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n, ptr %p0, ptr %p1) #0 {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
+; CHECK-SVE-NEXT: ptrue p1.h
+; CHECK-SVE-NEXT: punpklo p2.h, p0.b
+; CHECK-SVE-NEXT: punpkhi p0.h, p0.b
+; CHECK-SVE-NEXT: and p2.b, p2/z, p2.b, p1.b
+; CHECK-SVE-NEXT: and p0.b, p0/z, p0.b, p1.b
+; CHECK-SVE-NEXT: str p2, [x2]
+; CHECK-SVE-NEXT: str p0, [x3]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: mov w8, w1
+; CHECK-SVE2p1-NEXT: mov w9, w0
+; CHECK-SVE2p1-NEXT: whilelo { p0.h, p1.h }, x9, x8
+; CHECK-SVE2p1-NEXT: str p0, [x2]
+; CHECK-SVE2p1-NEXT: str p1, [x3]
+; CHECK-SVE2p1-NEXT: ret
+;
+; CHECK-SME2-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SME2: // %bb.0:
+; CHECK-SME2-NEXT: mov w8, w1
+; CHECK-SME2-NEXT: mov w9, w0
+; CHECK-SME2-NEXT: whilelo { p0.h, p1.h }, x9, x8
+; CHECK-SME2-NEXT: str p0, [x2]
+; CHECK-SME2-NEXT: str p1, [x3]
+; CHECK-SME2-NEXT: ret
+ %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 %i, i32 %n)
+ %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+ %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+ %pg0 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %v0)
+ %pg1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %v1)
+ store <vscale x 16 x i1> %pg0, ptr %p0
+ store <vscale x 16 x i1> %pg1, ptr %p1
+ ret void
+}
+
+define void @test_2x8bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n, ptr %p0, ptr %p1) #0 {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: whilelo p0.b, x0, x1
+; CHECK-SVE-NEXT: ptrue p1.h
+; CHECK-SVE-NEXT: punpklo p2.h, p0.b
+; CHECK-SVE-NEXT: punpkhi p0.h, p0.b
+; CHECK-SVE-NEXT: and p2.b, p2/z, p2.b, p1.b
+; CHECK-SVE-NEXT: and p0.b, p0/z, p0.b, p1.b
+; CHECK-SVE-NEXT: str p2, [x2]
+; CHECK-SVE-NEXT: str p0, [x3]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: whilelo { p0.h, p1.h }, x0, x1
+; CHECK-SVE2p1-NEXT: str p0, [x2]
+; CHECK-SVE2p1-NEXT: str p1, [x3]
+; CHECK-SVE2p1-NEXT: ret
+;
+; CHECK-SME2-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SME2: // %bb.0:
+; CHECK-SME2-NEXT: whilelo { p0.h, p1.h }, x0, x1
+; CHECK-SME2-NEXT: str p0, [x2]
+; CHECK-SME2-NEXT: str p1, [x3]
+; CHECK-SME2-NEXT: ret
+ %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+ %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+ %pg0 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %v0)
+ %pg1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> %v1)
+ store <vscale x 16 x i1> %pg0, ptr %p0
+ store <vscale x 16 x i1> %pg1, ptr %p1
+ ret void
+}
+
+define void @test_edge_case_2x1bit_mask(i64 %i, i64 %n, ptr %p0, ptr %p1) #0 {
+; CHECK-SVE-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: whilelo p1.d, x0, x1
+; CHECK-SVE-NEXT: punpklo p0.h, p0.b
+; CHECK-SVE-NEXT: punpklo p2.h, p1.b
+; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: and p2.b, p2/z, p2.b, p0.b
+; CHECK-SVE-NEXT: and p0.b, p1/z, p1.b, p0.b
+; CHECK-SVE-NEXT: str p2, [x2]
+; CHECK-SVE-NEXT: str p0, [x3]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: ptrue p0.d
+; CHECK-SVE2p1-NEXT: whilelo p1.d, x0, x1
+; CHECK-SVE2p1-NEXT: punpklo p0.h, p0.b
+; CHECK-SVE2p1-NEXT: punpklo p2.h, p1.b
+; CHECK-SVE2p1-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE2p1-NEXT: and p2.b, p2/z, p2.b, p0.b
+; CHECK-SVE2p1-NEXT: and p0.b, p1/z, p1.b, p0.b
+; CHECK-SVE2p1-NEXT: str p2, [x2]
+; CHECK-SVE2p1-NEXT: str p0, [x3]
+; CHECK-SVE2p1-NEXT: ret
+;
+; CHECK-SME2-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SME2: // %bb.0:
+; CHECK-SME2-NEXT: ptrue p0.d
+; CHECK-SME2-NEXT: whilelo p1.d, x0, x1
+; CHECK-SME2-NEXT: punpklo p0.h, p0.b
+; CHECK-SME2-NEXT: punpklo p2.h, p1.b
+; CHECK-SME2-NEXT: punpkhi p1.h, p1.b
+; CHECK-SME2-NEXT: and p2.b, p2/z, p2.b, p0.b
+; CHECK-SME2-NEXT: and p0.b, p1/z, p1.b, p0.b
+; CHECK-SME2-NEXT: str p2, [x2]
+; CHECK-SME2-NEXT: str p0, [x3]
+; CHECK-SME2-NEXT: ret
+ %r = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 0)
+ %v1 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 1)
+ %pg0 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv1i1(<vscale x 1 x i1> %v0)
+ %pg1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv1i1(<vscale x 1 x i1> %v1)
+ store <vscale x 16 x i1> %pg0, ptr %p0
+ store <vscale x 16 x i1> %pg1, ptr %p1
+ ret void
+}
+
+define void @test_edge_case_2x16bit_mask(i64 %i, i64 %n, ptr %p0, ptr %p1) #0 {
+; CHECK-SVE-LABEL: test_edge_case_2x16bit_mask:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: index z1.d, #0, #1
+; CHECK-SVE-NEXT: mov z0.d, x0
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: mov z23.d, x1
+; CHECK-SVE-NEXT: mov z2.d, z1.d
+; CHECK-SVE-NEXT: mov z3.d, z1.d
+; CHECK-SVE-NEXT: mov z4.d, z1.d
+; CHECK-SVE-NEXT: uqadd z5.d, z1.d, z0.d
+; CHECK-SVE-NEXT: incd z1.d, all, mul #8
+; CHECK-SVE-NEXT: incd z2.d
+; CHECK-SVE-NEXT: incd z3.d, all, mul #2
+; CHECK-SVE-NEXT: incd z4.d, all, mul #4
+; CHECK-SVE-NEXT: uqadd z1.d, z1.d, z0.d
+; CHECK-SVE-NEXT: cmphi p3.d, p0/z, z23.d, z5.d
+; CHECK-SVE-NEXT: mov z6.d, z2.d
+; CHECK-SVE-NEXT: mov z7.d, z2.d
+; CHECK-SVE-NEXT: mov z17.d, z3.d
+; CHECK-SVE-NEXT: uqadd z16.d, z2.d, z0.d
+; CHECK-SVE-NEXT: mov z19.d, z4.d
+; CHECK-SVE-NEXT: incd z2.d, all, mul #8
+; CHECK-SVE-NEXT: uqadd z18.d, z3.d, z0.d
+; CHECK-SVE-NEXT: incd z3.d, all, mul #8
+; CHECK-SVE-NEXT: cmphi p1.d, p0/z, z23.d, z1.d
+; CHECK-SVE-NEXT: incd z6.d, all, mul #2
+; CHECK-SVE-NEXT: incd z7.d, all, mul #4
+; CHECK-SVE-NEXT: incd z17.d, all, mul #4
+; CHECK-SVE-NEXT: incd z19.d, all, mul #8
+; CHECK-SVE-NEXT: uqadd z4.d, z4.d, z0.d
+; CHECK-SVE-NEXT: uqadd z2.d, z2.d, z0.d
+; CHECK-SVE-NEXT: cmphi p6.d, p0/z, z23.d, z16.d
+; CHECK-SVE-NEXT: uqadd z3.d, z3.d, z0.d
+; CHECK-SVE-NEXT: mov z20.d, z6.d
+; CHECK-SVE-NEXT: mov z21.d, z6.d
+; CHECK-SVE-NEXT: mov z22.d, z7.d
+; CHECK-SVE-NEXT: mov z24.d, z17.d
+; CHECK-SVE-NEXT: uqadd z6.d, z6.d, z0.d
+; CHECK-SVE-NEXT: uqadd z19.d, z19.d, z0.d
+; CHECK-SVE-NEXT: cmphi p2.d, p0/z, z23.d, z2.d
+; CHECK-SVE-NEXT: uqadd z7.d, z7.d, z0.d
+; CHECK-SVE-NEXT: uqadd z17.d, z17.d, z0.d
+; CHECK-SVE-NEXT: incd z20.d, all, mul #4
+; CHECK-SVE-NEXT: incd z21.d, all, mul #8
+; CHECK-SVE-NEXT: incd z22.d, all, mul #8
+; CHECK-SVE-NEXT: incd z24.d, all, mul #8
+; CHECK-SVE-NEXT: cmphi p4.d, p0/z, z23.d, z3.d
+; CHECK-SVE-NEXT: uzp1 p3.s, p3.s, p6.s
+; CHECK-SVE-NEXT: cmphi p7.d, p0/z, z23.d, z19.d
+; CHECK-SVE-NEXT: uzp1 p1.s, p1.s, p2.s
+; CHECK-SVE-NEXT: cmphi p2.d, p0/z, z23.d, z18.d
+; CHECK-SVE-NEXT: cmphi p9.d, p0/z, z23.d, z6.d
+; CHECK-SVE-NEXT: cmphi p6.d, p0/z, z23.d, z17.d
+; CHECK-SVE-NEXT: mov z25.d, z20.d
+; CHECK-SVE-NEXT: uqadd z21.d, z21.d, z0.d
+; CHECK-SVE-NEXT: uqadd z22.d, z22.d, z0.d
+; CHECK-SVE-NEXT: uqadd z20.d, z20.d, z0.d
+; CHECK-SVE-NEXT: uqadd z1.d, z24.d, z0.d
+; CHECK-SVE-NEXT: uzp1 p2.s, p2.s, p9.s
+; CHECK-SVE-NEXT: incd z25.d, all, mul #8
+; CHECK-SVE-NEXT: cmphi p5.d, p0/z, z23.d, z21.d
+; CHECK-SVE-NEXT: cmphi p8.d, p0/z, z23.d, z22.d
+; CHECK-SVE-NEXT: cmphi p10.d, p0/z, z23.d, z20.d
+; CHECK-SVE-NEXT: cmphi p9.d, p0/z, z23.d, z1.d
+; CHECK-SVE-NEXT: uqadd z0.d, z25.d, z0.d
+; CHECK-SVE-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-SVE-NEXT: uzp1 p5.s, p7.s, p8.s
+; CHECK-SVE-NEXT: cmphi p7.d, p0/z, z23.d, z4.d
+; CHECK-SVE-NEXT: cmphi p8.d, p0/z, z23.d, z7.d
+; CHECK-SVE-NEXT: uzp1 p6.s, p6.s, p10.s
+; CHECK-SVE-NEXT: uzp1 p2.h, p3.h, p2.h
+; CHECK-SVE-NEXT: cmphi p0.d, p0/z, z23.d, z0.d
+; CHECK-SVE-NEXT: uzp1 p7.s, p7.s, p8.s
+; CHECK-SVE-NEXT: uzp1 p1.h, p1.h, p4.h
+; CHECK-SVE-NEXT: uzp1 p0.s, p9.s, p0.s
+; CHECK-SVE-NEXT: uzp1 p3.h, p7.h, p6.h
+; CHECK-SVE-NEXT: uzp1 p0.h, p5.h, p0.h
+; CHECK-SVE-NEXT: uzp1 p2.b, p2.b, p3.b
+; CHECK-SVE-NEXT: uzp1 p0.b, p1.b, p0.b
+; CHECK-SVE-NEXT: str p2, [x2]
+; CHECK-SVE-NEXT: str p0, [x3]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: test_edge_case_2x16bit_mask:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: index z1.d, #0, #1
+; CHECK-SVE2p1-NEXT: mov z0.d, x0
+; CHECK-SVE2p1-NEXT: ptrue p0.d
+; CHECK-SVE2p1-NEXT: mov z23.d, x1
+; CHECK-SVE2p1-NEXT: mov z2.d, z1.d
+; CHECK-SVE2p1-NEXT: mov z3.d, z1.d
+; CHECK-SVE2p1-NEXT: mov z4.d, z1.d
+; CHECK-SVE2p1-NEXT: uqadd z5.d, z1.d, z0.d
+; CHECK-SVE2p1-NEXT: incd z1.d, all, mul #8
+; CHECK-SVE2p1-NEXT: incd z2.d
+; CHECK-SVE2p1-NEXT: incd z3.d, all, mul #2
+; CHECK-SVE2p1-NEXT: incd z4.d, all, mul #4
+; CHECK-SVE2p1-NEXT: uqadd z1.d, z1.d, z0.d
+; CHECK-SVE2p1-NEXT: cmphi p3.d, p0/z, z23.d, z5.d
+; CHECK-SVE2p1-NEXT: mov z6.d, z2.d
+; CHECK-SVE2p1-NEXT: mov z7.d, z2.d
+; CHECK-SVE2p1-NEXT: mov z17.d, z3.d
+; CHECK-SVE2p1-NEXT: uqadd z16.d, z2.d, z0.d
+; CHECK-SVE2p1-NEXT: mov z19.d, z4.d
+; CHECK-SVE2p1-NEXT: incd z2.d, all, mul #8
+; CHECK-SVE2p1-NEXT: uqadd z18.d, z3.d, z0.d
+; CHECK-SVE2p1-NEXT: incd z3.d, all, mul #8
+; CHECK-SVE2p1-NEXT: cmphi p1.d, p0/z, z23.d, z1.d
+; CHECK-SVE2p1-NEXT: incd z6.d, all, mul #2
+; CHECK-SVE2p1-NEXT: incd z7.d, all, mul #4
+; CHECK-SVE2p1-NEXT: incd z17.d, all, mul #4
+; CHECK-SVE2p1-NEXT: incd z19.d, all, mul #8
+; CHECK-SVE2p1-NEXT: uqadd z4.d, z4.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z2.d, z2.d, z0.d
+; CHECK-SVE2p1-NEXT: cmphi p6.d, p0/z, z23.d, z16.d
+; CHECK-SVE2p1-NEXT: uqadd z3.d, z3.d, z0.d
+; CHECK-SVE2p1-NEXT: mov z20.d, z6.d
+; CHECK-SVE2p1-NEXT: mov z21.d, z6.d
+; CHECK-SVE2p1-NEXT: mov z22.d, z7.d
+; CHECK-SVE2p1-NEXT: mov z24.d, z17.d
+; CHECK-SVE2p1-NEXT: uqadd z6.d, z6.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z19.d, z19.d, z0.d
+; CHECK-SVE2p1-NEXT: cmphi p2.d, p0/z, z23.d, z2.d
+; CHECK-SVE2p1-NEXT: uqadd z7.d, z7.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z17.d, z17.d, z0.d
+; CHECK-SVE2p1-NEXT: incd z20.d, all, mul #4
+; CHECK-SVE2p1-NEXT: incd z21.d, all, mul #8
+; CHECK-SVE2p1-NEXT: incd z22.d, all, mul #8
+; CHECK-SVE2p1-NEXT: incd z24.d, all, mul #8
+; CHECK-SVE2p1-NEXT: cmphi p4.d, p0/z, z23.d, z3.d
+; CHECK-SVE2p1-NEXT: uzp1 p3.s, p3.s, p6.s
+; CHECK-SVE2p1-NEXT: cmphi p7.d, p0/z, z23.d, z19.d
+; CHECK-SVE2p1-NEXT: uzp1 p1.s, p1.s, p2.s
+; CHECK-SVE2p1-NEXT: cmphi p2.d, p0/z, z23.d, z18.d
+; CHECK-SVE2p1-NEXT: cmphi p9.d, p0/z, z23.d, z6.d
+; CHECK-SVE2p1-NEXT: cmphi p6.d, p0/z, z23.d, z17.d
+; CHECK-SVE2p1-NEXT: mov z25.d, z20.d
+; CHECK-SVE2p1-NEXT: uqadd z21.d, z21.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z22.d, z22.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z20.d, z20.d, z0.d
+; CHECK-SVE2p1-NEXT: uqadd z1.d, z24.d, z0.d
+; CHECK-SVE2p1-NEXT: uzp1 p2.s, p2.s, p9.s
+; CHECK-SVE2p1-NEXT: incd z25.d, all, mul #8
+; CHECK-SVE2p1-NEXT: cmphi p5.d, p0/z, z23.d, z21.d
+; CHECK-SVE2p1-NEXT: cmphi p8.d, p0/z, z23.d, z22.d
+; CHECK-SVE2p1-NEXT: cmphi p10.d, p0/z, z23.d, z20.d
+; CHECK-SVE2p1-NEXT: cmphi p9.d, p0/z, z23.d, z1.d
+; CHECK-SVE2p1-NEXT: uqadd z0.d, z25.d, z0.d
+; CHECK-SVE2p1-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-SVE2p1-NEXT: uzp1 p5.s, p7.s, p8.s
+; CHECK-SVE2p1-NEXT: cmphi p7.d, p0/z, z23.d, z4.d
+; CHECK-SVE2p1-NEXT: cmphi p8.d, p0/z, z23.d, z7.d
+; CHECK-SVE2p1-NEXT: uzp1 p6.s, p6.s, p10.s
+; CHECK-SVE2p1-NEXT: uzp1 p2.h, p3.h, p2.h
+; CHECK-SVE2p1-NEXT: cmphi p0.d, p0/z, z23.d, z0.d
+; CHECK-SVE2p1-NEXT: uzp1 p7.s, p7.s, p8.s
+; CHECK-SVE2p1-NEXT: uzp1 p1.h, p1.h, p4.h
+; CHECK-SVE2p1-NEXT: uzp1 p0.s, p9.s, p0.s
+; CHECK-SVE2p1-NEXT: uzp1 p3.h, p7.h, p6.h
+; CHECK-SVE2p1-NEXT: uzp1 p0.h, p5.h, p0.h
+; CHECK-SVE2p1-NEXT: uzp1 p2.b, p2.b, p3.b
+; CHECK-SVE2p1-NEXT: uzp1 p0.b, p1.b, p0.b
+; CHECK-SVE2p1-NEXT: str p2, [x2]
+; CHECK-SVE2p1-NEXT: str p0, [x3]
+; CHECK-SVE2p1-NEXT: ret
+;
+; CHECK-SME2-LABEL: test_edge_case_2x16bit_mask:
+; CHECK-SME2: // %bb.0:
+; CHECK-SME2-NEXT: index z1.d, #0, #1
+; CHECK-SME2-NEXT: mov z0.d, x0
+; CHECK-SME2-NEXT: ptrue p0.d
+; CHECK-SME2-NEXT: mov z23.d, x1
+; CHECK-SME2-NEXT: mov z2.d, z1.d
+; CHECK-SME2-NEXT: mov z3.d, z1.d
+; CHECK-SME2-NEXT: mov z4.d, z1.d
+; CHECK-SME2-NEXT: uqadd z5.d, z1.d, z0.d
+; CHECK-SME2-NEXT: incd z1.d, all, mul #8
+; CHECK-SME2-NEXT: incd z2.d
+; CHECK-SME2-NEXT: incd z3.d, all, mul #2
+; CHECK-SME2-NEXT: incd z4.d, all, mul #4
+; CHECK-SME2-NEXT: uqadd z1.d, z1.d, z0.d
+; CHECK-SME2-NEXT: cmphi p3.d, p0/z, z23.d, z5.d
+; CHECK-SME2-NEXT: mov z6.d, z2.d
+; CHECK-SME2-NEXT: mov z7.d, z2.d
+; CHECK-SME2-NEXT: mov z17.d, z3.d
+; CHECK-SME2-NEXT: uqadd z16.d, z2.d, z0.d
+; CHECK-SME2-NEXT: mov z19.d, z4.d
+; CHECK-SME2-NEXT: incd z2.d, all, mul #8
+; CHECK-SME2-NEXT: uqadd z18.d, z3.d, z0.d
+; CHECK-SME2-NEXT: incd z3.d, all, mul #8
+; CHECK-SME2-NEXT: cmphi p1.d, p0/z, z23.d, z1.d
+; CHECK-SME2-NEXT: incd z6.d, all, mul #2
+; CHECK-SME2-NEXT: incd z7.d, all, mul #4
+; CHECK-SME2-NEXT: incd z17.d, all, mul #4
+; CHECK-SME2-NEXT: incd z19.d, all, mul #8
+; CHECK-SME2-NEXT: uqadd z4.d, z4.d, z0.d
+; CHECK-SME2-NEXT: uqadd z2.d, z2.d, z0.d
+; CHECK-SME2-NEXT: cmphi p6.d, p0/z, z23.d, z16.d
+; CHECK-SME2-NEXT: uqadd z3.d, z3.d, z0.d
+; CHECK-SME2-NEXT: mov z20.d, z6.d
+; CHECK-SME2-NEXT: mov z21.d, z6.d
+; CHECK-SME2-NEXT: mov z22.d, z7.d
+; CHECK-SME2-NEXT: mov z24.d, z17.d
+; CHECK-SME2-NEXT: uqadd z6.d, z6.d, z0.d
+; CHECK-SME2-NEXT: uqadd z19.d, z19.d, z0.d
+; CHECK-SME2-NEXT: cmphi p2.d, p0/z, z23.d, z2.d
+; CHECK-SME2-NEXT: uqadd z7.d, z7.d, z0.d
+; CHECK-SME2-NEXT: uqadd z17.d, z17.d, z0.d
+; CHECK-SME2-NEXT: incd z20.d, all, mul #4
+; CHECK-SME2-NEXT: incd z21.d, all, mul #8
+; CHECK-SME2-NEXT: incd z22.d, all, mul #8
+; CHECK-SME2-NEXT: incd z24.d, all, mul #8
+; CHECK-SME2-NEXT: cmphi p4.d, p0/z, z23.d, z3.d
+; CHECK-SME2-NEXT: uzp1 p3.s, p3.s, p6.s
+; CHECK-SME2-NEXT: cmphi p7.d, p0/z, z23.d, z19.d
+; CHECK-SME2-NEXT: uzp1 p1.s, p1.s, p2.s
+; CHECK-SME2-NEXT: cmphi p2.d, p0/z, z23.d, z18.d
+; CHECK-SME2-NEXT: cmphi p9.d, p0/z, z23.d, z6.d
+; CHECK-SME2-NEXT: cmphi p6.d, p0/z, z23.d, z17.d
+; CHECK-SME2-NEXT: mov z25.d, z20.d
+; CHECK-SME2-NEXT: uqadd z21.d, z21.d, z0.d
+; CHECK-SME2-NEXT: uqadd z22.d, z22.d, z0.d
+; CHECK-SME2-NEXT: uqadd z20.d, z20.d, z0.d
+; CHECK-SME2-NEXT: uqadd z1.d, z24.d, z0.d
+; CHECK-SME2-NEXT: uzp1 p2.s, p2.s, p9.s
+; CHECK-SME2-NEXT: incd z25.d, all, mul #8
+; CHECK-SME2-NEXT: cmphi p5.d, p0/z, z23.d, z21.d
+; CHECK-SME2-NEXT: cmphi p8.d, p0/z, z23.d, z22.d
+; CHECK-SME2-NEXT: cmphi p10.d, p0/z, z23.d, z20.d
+; CHECK-SME2-NEXT: cmphi p9.d, p0/z, z23.d, z1.d
+; CHECK-SME2-NEXT: uqadd z0.d, z25.d, z0.d
+; CHECK-SME2-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-SME2-NEXT: uzp1 p5.s, p7.s, p8.s
+; CHECK-SME2-NEXT: cmphi p7.d, p0/z, z23.d, z4.d
+; CHECK-SME2-NEXT: cmphi p8.d, p0/z, z23.d, z7.d
+; CHECK-SME2-NEXT: uzp1 p6.s, p6.s, p10.s
+; CHECK-SME2-NEXT: uzp1 p2.h, p3.h, p2.h
+; CHECK-SME2-NEXT: cmphi p0.d, p0/z, z23.d, z0.d
+; CHECK-SME2-NEXT: uzp1 p7.s, p7.s, p8.s
+; CHECK-SME2-NEXT: uzp1 p1.h, p1.h, p4.h
+; CHECK-SME2-NEXT: uzp1 p0.s, p9.s, p0.s
+; CHECK-SME2-NEXT: uzp1 p3.h, p7.h, p6.h
+; CHECK-SME2-NEXT: uzp1 p0.h, p5.h, p0.h
+; CHECK-SME2-NEXT: uzp1 p2.b, p2.b, p3.b
+; CHECK-SME2-NEXT: uzp1 p0.b, p1.b, p0.b
+; CHECK-SME2-NEXT: str p2, [x2]
+; CHECK-SME2-NEXT: str p0, [x3]
+; CHECK-SME2-NEXT: ret
+ %r = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1.i64(<vscale x 32 x i1> %r, i64 0)
+ %v1 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1.i64(<vscale x 32 x i1> %r, i64 16)
+ store <vscale x 16 x i1> %v0, ptr %p0
+ store <vscale x 16 x i1> %v1, ptr %p1
+ ret void
+}
+
+define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n, ptr %p0, ptr %p1) #0 {
+; CHECK-SVE-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: whilelo p0.s, x0, x1
+; CHECK-SVE-NEXT: ptrue p1.d
+; CHECK-SVE-NEXT: punpklo p2.h, p0.b
+; CHECK-SVE-NEXT: punpkhi p0.h, p0.b
+; CHECK-SVE-NEXT: and p2.b, p2/z, p2.b, p1.b
+; CHECK-SVE-NEXT: and p0.b, p0/z, p0.b, p1.b
+; CHECK-SVE-NEXT: str p2, [x2]
+; CHECK-SVE-NEXT: str p0, [x3]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SVE2p1: // %bb.0:
+; CHECK-SVE2p1-NEXT: whilelo { p0.d, p1.d }, x0, x1
+; CHECK-SVE2p1-NEXT: str p0, [x2]
+; CHECK-SVE2p1-NEXT: str p1, [x3]
+; CHECK-SVE2p1-NEXT: ret
+;
+; CHECK-SME2-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SME2: // %bb.0:
+; CHECK-SME2-NEXT: whilelo { p0.d, p1.d }, x0, x1
+; CHECK-SME2-NEXT: str p0, [x2]
+; CHECK-SME2-NEXT: str p1, [x3]
+; CHECK-SME2-NEXT: ret
+ %r = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 0)
+ %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 2)
+ %pg0 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %v0)
+ %pg1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %v1)
+ store <vscale x 16 x i1> %pg0, ptr %p0
+ store <vscale x 16 x i1> %pg1, ptr %p1
+ ret void
+}
+
+attributes #0 = { nounwind }
More information about the llvm-commits
mailing list