[llvm] 5d4f6ce - [AArch64][SVE] Zero other lanes when doing OR reduction on unpacked predicate using ptest.
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 6 09:13:57 PDT 2022
Author: Sander de Smalen
Date: 2022-07-06T16:12:44Z
New Revision: 5d4f6ce22990a48d0d414b94e005ed4816c61261
URL: https://github.com/llvm/llvm-project/commit/5d4f6ce22990a48d0d414b94e005ed4816c61261
DIFF: https://github.com/llvm/llvm-project/commit/5d4f6ce22990a48d0d414b94e005ed4816c61261.diff
LOG: [AArch64][SVE] Zero other lanes when doing OR reduction on unpacked predicate using ptest.
When the predicate vector is unpacked, we cannot assume anything about the
values in the other lanes. We have to make sure we use the correct
predicate where we know that the other lanes have been zeroed.
Reviewed By: RosieSumpter
Differential Revision: https://reviews.llvm.org/D129081
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll
llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 73e129c7d89d..105a7c868738 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -21094,7 +21094,7 @@ SDValue AArch64TargetLowering::LowerPredReductionToSVE(SDValue ReduceOp,
default:
return SDValue();
case ISD::VECREDUCE_OR:
- if (isAllActivePredicate(DAG, Pg))
+ if (isAllActivePredicate(DAG, Pg) && OpVT == MVT::nxv16i1)
// The predicate can be 'Op' because
// vecreduce_or(Op & <all true>) <=> vecreduce_or(Op).
return getPTest(DAG, VT, Op, Op, AArch64CC::ANY_ACTIVE);
diff --git a/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll
index bd129cf2c0a7..fe4b6ca0f066 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-pred-reduce.ll
@@ -79,7 +79,8 @@ define i1 @reduce_or_nxv16i1(<vscale x 16 x i1> %vec) {
define i1 @reduce_or_nxv8i1(<vscale x 8 x i1> %vec) {
; CHECK-LABEL: reduce_or_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.or.i1.nxv8i1(<vscale x 8 x i1> %vec)
@@ -89,7 +90,8 @@ define i1 @reduce_or_nxv8i1(<vscale x 8 x i1> %vec) {
define i1 @reduce_or_nxv4i1(<vscale x 4 x i1> %vec) {
; CHECK-LABEL: reduce_or_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.or.i1.nxv4i1(<vscale x 4 x i1> %vec)
@@ -99,7 +101,8 @@ define i1 @reduce_or_nxv4i1(<vscale x 4 x i1> %vec) {
define i1 @reduce_or_nxv2i1(<vscale x 2 x i1> %vec) {
; CHECK-LABEL: reduce_or_nxv2i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.or.i1.nxv2i1(<vscale x 2 x i1> %vec)
@@ -109,7 +112,9 @@ define i1 @reduce_or_nxv2i1(<vscale x 2 x i1> %vec) {
define i1 @reduce_or_nxv1i1(<vscale x 1 x i1> %vec) {
; CHECK-LABEL: reduce_or_nxv1i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.or.i1.nxv1i1(<vscale x 1 x i1> %vec)
@@ -252,7 +257,8 @@ define i1 @reduce_smin_nxv16i1(<vscale x 16 x i1> %vec) {
define i1 @reduce_smin_nxv8i1(<vscale x 8 x i1> %vec) {
; CHECK-LABEL: reduce_smin_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.smin.i1.nxv8i1(<vscale x 8 x i1> %vec)
@@ -262,7 +268,8 @@ define i1 @reduce_smin_nxv8i1(<vscale x 8 x i1> %vec) {
define i1 @reduce_smin_nxv4i1(<vscale x 4 x i1> %vec) {
; CHECK-LABEL: reduce_smin_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.smin.i1.nxv4i1(<vscale x 4 x i1> %vec)
@@ -272,7 +279,8 @@ define i1 @reduce_smin_nxv4i1(<vscale x 4 x i1> %vec) {
define i1 @reduce_smin_nxv2i1(<vscale x 2 x i1> %vec) {
; CHECK-LABEL: reduce_smin_nxv2i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.smin.i1.nxv2i1(<vscale x 2 x i1> %vec)
@@ -282,7 +290,9 @@ define i1 @reduce_smin_nxv2i1(<vscale x 2 x i1> %vec) {
define i1 @reduce_smin_nxv1i1(<vscale x 1 x i1> %vec) {
; CHECK-LABEL: reduce_smin_nxv1i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.smin.i1.nxv1i1(<vscale x 1 x i1> %vec)
@@ -304,7 +314,8 @@ define i1 @reduce_umax_nxv16i1(<vscale x 16 x i1> %vec) {
define i1 @reduce_umax_nxv8i1(<vscale x 8 x i1> %vec) {
; CHECK-LABEL: reduce_umax_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.umax.i1.nxv8i1(<vscale x 8 x i1> %vec)
@@ -314,7 +325,8 @@ define i1 @reduce_umax_nxv8i1(<vscale x 8 x i1> %vec) {
define i1 @reduce_umax_nxv4i1(<vscale x 4 x i1> %vec) {
; CHECK-LABEL: reduce_umax_nxv4i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.umax.i1.nxv4i1(<vscale x 4 x i1> %vec)
@@ -324,7 +336,8 @@ define i1 @reduce_umax_nxv4i1(<vscale x 4 x i1> %vec) {
define i1 @reduce_umax_nxv2i1(<vscale x 2 x i1> %vec) {
; CHECK-LABEL: reduce_umax_nxv2i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.umax.i1.nxv2i1(<vscale x 2 x i1> %vec)
@@ -334,7 +347,9 @@ define i1 @reduce_umax_nxv2i1(<vscale x 2 x i1> %vec) {
define i1 @reduce_umax_nxv1i1(<vscale x 1 x i1> %vec) {
; CHECK-LABEL: reduce_umax_nxv1i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%res = call i1 @llvm.vector.reduce.umax.i1.nxv1i1(<vscale x 1 x i1> %vec)
diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
index 425fbf8983dd..f46c5fa4e2fa 100644
--- a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
@@ -8,7 +8,8 @@
define i1 @reduce_or_insert_subvec_into_zero(<vscale x 4 x i1> %in) {
; CHECK-LABEL: reduce_or_insert_subvec_into_zero:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%t = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> zeroinitializer, <vscale x 4 x i1> %in, i64 0)
@@ -19,7 +20,8 @@ define i1 @reduce_or_insert_subvec_into_zero(<vscale x 4 x i1> %in) {
define i1 @reduce_or_insert_subvec_into_poison(<vscale x 4 x i1> %in) {
; CHECK-LABEL: reduce_or_insert_subvec_into_poison:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptest p1, p0.b
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%t = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> poison, <vscale x 4 x i1> %in, i64 0)
More information about the llvm-commits
mailing list