[llvm] [AArch64] replace SVE intrinsics with no active lanes with zero (PR #107413)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 5 08:09:02 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: None (Lukacma)
<details>
<summary>Changes</summary>
This patch extends https://github.com/llvm/llvm-project/pull/73964 and optimises SVE intrinsics into zero constants when predicate is zero.
---
Full diff: https://github.com/llvm/llvm-project/pull/107413.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+34-10)
- (added) llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-to-zero.ll (+232)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 58c267f1ce4bd6..9a887b832c90f6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1110,10 +1110,10 @@ instCombineSVENoActiveUnaryErase(InstCombiner &IC, IntrinsicInst &II,
return std::nullopt;
}
-// Simplify unary operation where predicate has all inactive lanes by replacing
+// Simplify operation where predicate has all inactive lanes by replacing
// instruction with zeroed object
static std::optional<Instruction *>
-instCombineSVENoActiveUnaryZero(InstCombiner &IC, IntrinsicInst &II) {
+instCombineSVENoActiveZero(InstCombiner &IC, IntrinsicInst &II) {
if (match(II.getOperand(0), m_ZeroInt())) {
Constant *Node;
Type *RetTy = II.getType();
@@ -1126,10 +1126,9 @@ instCombineSVENoActiveUnaryZero(InstCombiner &IC, IntrinsicInst &II) {
: ConstantInt::get(VecT, 0));
}
Node = ConstantStruct::get(StructT, ZerVec);
- } else if (RetTy->isFPOrFPVectorTy())
- Node = ConstantFP::get(RetTy, 0.0);
- else
- Node = ConstantInt::get(II.getType(), 0);
+ } else
+ Node = RetTy->isFPOrFPVectorTy() ? ConstantFP::get(RetTy, 0.0)
+ : ConstantInt::get(II.getType(), 0);
IC.replaceInstUsesWith(II, Node);
return IC.eraseInstFromFunction(II);
@@ -1188,7 +1187,7 @@ static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
LLVMContext &Ctx = II.getContext();
// Replace by zero constant when all lanes are inactive
- if (auto II_NA = instCombineSVENoActiveUnaryZero(IC, II))
+ if (auto II_NA = instCombineSVENoActiveZero(IC, II))
return II_NA;
// Check that the predicate is all active
@@ -1556,7 +1555,7 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
Type *VecTy = II.getType();
// Replace by zero constant when all lanes are inactive
- if (auto II_NA = instCombineSVENoActiveUnaryZero(IC, II))
+ if (auto II_NA = instCombineSVENoActiveZero(IC, II))
return II_NA;
if (isAllActivePredicate(Pred)) {
@@ -1907,7 +1906,7 @@ instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
Value *PassThru = ConstantAggregateZero::get(Ty);
// Replace by zero constant when all lanes are inactive
- if (auto II_NA = instCombineSVENoActiveUnaryZero(IC, II))
+ if (auto II_NA = instCombineSVENoActiveZero(IC, II))
return II_NA;
// Contiguous gather => masked load.
@@ -2197,6 +2196,31 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
case Intrinsic::aarch64_sve_st4:
case Intrinsic::aarch64_sve_st4q:
return instCombineSVENoActiveUnaryErase(IC, II, 4);
+ case Intrinsic::aarch64_sve_addqv:
+ case Intrinsic::aarch64_sve_and_z:
+ case Intrinsic::aarch64_sve_bic_z:
+ case Intrinsic::aarch64_sve_brka_z:
+ case Intrinsic::aarch64_sve_brkb_z:
+ case Intrinsic::aarch64_sve_brkn_z:
+ case Intrinsic::aarch64_sve_brkpa_z:
+ case Intrinsic::aarch64_sve_brkpb_z:
+ case Intrinsic::aarch64_sve_cntp:
+ case Intrinsic::aarch64_sve_compact:
+ case Intrinsic::aarch64_sve_eor_z:
+ case Intrinsic::aarch64_sve_eorv:
+ case Intrinsic::aarch64_sve_eorqv:
+ case Intrinsic::aarch64_sve_nand_z:
+ case Intrinsic::aarch64_sve_nor_z:
+ case Intrinsic::aarch64_sve_orn_z:
+ case Intrinsic::aarch64_sve_orr_z:
+ case Intrinsic::aarch64_sve_orv:
+ case Intrinsic::aarch64_sve_orqv:
+ case Intrinsic::aarch64_sve_pnext:
+ case Intrinsic::aarch64_sve_rdffr_z:
+ case Intrinsic::aarch64_sve_saddv:
+ case Intrinsic::aarch64_sve_uaddv:
+ case Intrinsic::aarch64_sve_umaxv:
+ case Intrinsic::aarch64_sve_umaxqv:
case Intrinsic::aarch64_sve_cmpeq:
case Intrinsic::aarch64_sve_cmpeq_wide:
case Intrinsic::aarch64_sve_cmpge:
@@ -2251,7 +2275,7 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
case Intrinsic::aarch64_sve_ldnt1_gather_index:
case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
- return instCombineSVENoActiveUnaryZero(IC, II);
+ return instCombineSVENoActiveZero(IC, II);
case Intrinsic::aarch64_sve_prf:
case Intrinsic::aarch64_sve_prfb_gather_index:
case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-to-zero.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-to-zero.ll
new file mode 100644
index 00000000000000..e58aa2eeefa8d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-to-zero.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+;RUN: opt -S -passes=instcombine < %s | FileCheck %s
+target triple = "aarch64-unknown-linux-gnu"
+
+
+define <16 x i8> @addqv_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: define <16 x i8> @addqv_i8(
+; CHECK-SAME: <vscale x 16 x i8> [[A:%.*]]) {
+; CHECK-NEXT: ret <16 x i8> zeroinitializer
+;
+ %res = call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i8> %a);
+ ret <16 x i8> %res
+}
+
+define <vscale x 4 x i1> @and_4(<vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 4 x i1> @and_4(
+; CHECK-SAME: <vscale x 4 x i1> [[PN:%.*]], <vscale x 4 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 4 x i1> zeroinitializer
+;
+ %res = call <vscale x 4 x i1> @llvm.aarch64.sve.and.z.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+ ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 16 x i1> @bic_16(<vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 16 x i1> @bic_16(
+; CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %res = call <vscale x 16 x i1> @llvm.aarch64.sve.bic.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+ ret <vscale x 16 x i1> %res;
+}
+
+define <vscale x 16 x i1> @brka_z_b8(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: define <vscale x 16 x i1> @brka_z_b8(
+; CHECK-SAME: <vscale x 16 x i1> [[A:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.brka.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @brkb_z_b8(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: define <vscale x 16 x i1> @brkb_z_b8(
+; CHECK-SAME: <vscale x 16 x i1> [[A:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.brkb.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %a)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @brkn_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: define <vscale x 16 x i1> @brkn_b8(
+; CHECK-SAME: <vscale x 16 x i1> [[A:%.*]], <vscale x 16 x i1> [[B:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.brkn.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @brkpa_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: define <vscale x 16 x i1> @brkpa_b8(
+; CHECK-SAME: <vscale x 16 x i1> [[A:%.*]], <vscale x 16 x i1> [[B:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.brkpa.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
+ ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @brkpb_b8(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: define <vscale x 16 x i1> @brkpb_b8(
+; CHECK-SAME: <vscale x 16 x i1> [[A:%.*]], <vscale x 16 x i1> [[B:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.brkpb.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
+ ret <vscale x 16 x i1> %out
+}
+
+define i64 @cntp_b64(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: define i64 @cntp_b64(
+; CHECK-SAME: <vscale x 2 x i1> [[A:%.*]]) {
+; CHECK-NEXT: ret i64 0
+;
+; USE_SCALAR_INC-LABEL: cntp_b64:
+; USE_SCALAR_INC: // %bb.0:
+; USE_SCALAR_INC-NEXT: cntp x0, p0, p1.d
+; USE_SCALAR_INC-NEXT: ret
+ %out = call i64 @llvm.aarch64.sve.cntp.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> %a)
+ ret i64 %out
+}
+
+define <vscale x 4 x i32> @compact_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: define <vscale x 4 x i32> @compact_i32(
+; CHECK-SAME: <vscale x 4 x i32> [[A:%.*]]) {
+; CHECK-NEXT: ret <vscale x 4 x i32> zeroinitializer
+;
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.compact.nxv4i32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %a)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 16 x i1> @eor_16(<vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 16 x i1> @eor_16(
+; CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %res = call <vscale x 16 x i1> @llvm.aarch64.sve.eor.z.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> %Pn, <vscale x 16 x i1> %Pd)
+ ret <vscale x 16 x i1> %res;
+}
+
+define i32 @eorv_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: define i32 @eorv_i32(
+; CHECK-SAME: <vscale x 4 x i32> [[A:%.*]]) {
+; CHECK-NEXT: ret i32 0
+;
+ %out = call i32 @llvm.aarch64.sve.eorv.nxv4i32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %a)
+ ret i32 %out
+}
+
+define <4 x i32> @eorqv_i32(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: define <4 x i32> @eorqv_i32(
+; CHECK-SAME: <vscale x 4 x i32> [[A:%.*]]) {
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %res = call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %a);
+ ret <4 x i32> %res
+}
+
+define <vscale x 8 x i1> @nand_8(<vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 8 x i1> @nand_8(
+; CHECK-SAME: <vscale x 8 x i1> [[PN:%.*]], <vscale x 8 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 8 x i1> zeroinitializer
+;
+ %res = call <vscale x 8 x i1> @llvm.aarch64.sve.nand.z.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> %Pn, <vscale x 8 x i1> %Pd)
+ ret <vscale x 8 x i1> %res;
+}
+
+define <vscale x 4 x i1> @nor_4(<vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 4 x i1> @nor_4(
+; CHECK-SAME: <vscale x 4 x i1> [[PN:%.*]], <vscale x 4 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 4 x i1> zeroinitializer
+;
+ %res = call <vscale x 4 x i1> @llvm.aarch64.sve.nor.z.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+ ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 4 x i1> @orn_4(<vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 4 x i1> @orn_4(
+; CHECK-SAME: <vscale x 4 x i1> [[PN:%.*]], <vscale x 4 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 4 x i1> zeroinitializer
+;
+ %res = call <vscale x 4 x i1> @llvm.aarch64.sve.orn.z.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> %Pn, <vscale x 4 x i1> %Pd)
+ ret <vscale x 4 x i1> %res;
+}
+
+define <vscale x 2 x i1> @orr_2(<vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd) {
+; CHECK-LABEL: define <vscale x 2 x i1> @orr_2(
+; CHECK-SAME: <vscale x 2 x i1> [[PN:%.*]], <vscale x 2 x i1> [[PD:%.*]]) {
+; CHECK-NEXT: ret <vscale x 2 x i1> zeroinitializer
+;
+ %res = call <vscale x 2 x i1> @llvm.aarch64.sve.orr.z.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> %Pn, <vscale x 2 x i1> %Pd)
+ ret <vscale x 2 x i1> %res;
+}
+
+define i8 @orv_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: define i8 @orv_i8(
+; CHECK-SAME: <vscale x 16 x i8> [[A:%.*]]) {
+; CHECK-NEXT: ret i8 0
+;
+ %out = call i8 @llvm.aarch64.sve.orv.nxv16i8(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i8> %a)
+ ret i8 %out
+}
+
+define <8 x i16> @orqv_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: define <8 x i16> @orqv_i16(
+; CHECK-SAME: <vscale x 8 x i16> [[A:%.*]]) {
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %res = call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i16> %a);
+ ret <8 x i16> %res
+}
+
+define <vscale x 4 x i1> @pnext_b32(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: define <vscale x 4 x i1> @pnext_b32(
+; CHECK-SAME: <vscale x 4 x i1> [[A:%.*]]) {
+; CHECK-NEXT: ret <vscale x 4 x i1> zeroinitializer
+;
+ %out = call <vscale x 4 x i1> @llvm.aarch64.sve.pnext.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> %a)
+ ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 16 x i1> @rdffr_z() {
+; CHECK-LABEL: define <vscale x 16 x i1> @rdffr_z() {
+; CHECK-NEXT: ret <vscale x 16 x i1> zeroinitializer
+;
+ %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> zeroinitializer)
+ ret <vscale x 16 x i1> %out
+}
+
+define i64 @saddv_i64(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: define i64 @saddv_i64(
+; CHECK-SAME: <vscale x 2 x i64> [[A:%.*]]) {
+; CHECK-NEXT: ret i64 0
+;
+ %out = call i64 @llvm.aarch64.sve.saddv.nxv2i64(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i64> %a)
+ ret i64 %out
+}
+
+define i64 @uaddv_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: define i64 @uaddv_i8(
+; CHECK-SAME: <vscale x 16 x i8> [[A:%.*]]) {
+; CHECK-NEXT: ret i64 0
+;
+ %out = call i64 @llvm.aarch64.sve.uaddv.nxv16i8(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i8> %a)
+ ret i64 %out
+}
+
+define i8 @umaxv_i8(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: define i8 @umaxv_i8(
+; CHECK-SAME: <vscale x 16 x i8> [[A:%.*]]) {
+; CHECK-NEXT: ret i8 0
+;
+ %out = call i8 @llvm.aarch64.sve.umaxv.nxv16i8(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i8> %a)
+ ret i8 %out
+}
+
+define <8 x i16> @umaxqv_i16(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: define <8 x i16> @umaxqv_i16(
+; CHECK-SAME: <vscale x 8 x i16> [[A:%.*]]) {
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %res = call <8 x i16> @llvm.aarch64.sve.umaxqv.v8i16.nxv8i16(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i16> %a);
+ ret <8 x i16> %res
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/107413
More information about the llvm-commits
mailing list