[llvm] [AArch64] Prevent unnecessary truncation in bool vector reduce code generation (PR #120096)
Csanád Hajdú via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 16 07:15:21 PST 2024
https://github.com/Il-Capitano created https://github.com/llvm/llvm-project/pull/120096
Prevent unnecessarily truncating results of 128 bit wide vector comparisons to 64 bit wide vector values in boolean vector reduce operations.
>From 64e808909181551e1b1f07956ea52efa7d6961d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu at arm.com>
Date: Mon, 16 Dec 2024 15:52:54 +0100
Subject: [PATCH] [AArch64] Prevent unnecessary truncation in bool vector
reduce code generation
Prevent unnecessarily truncating results of 128 bit wide vector
comparisons to 64 bit wide vector values in boolean vector reduce
operations.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 19 +-
.../illegal-floating-point-vector-compares.ll | 3 +-
llvm/test/CodeGen/AArch64/vecreduce-bool.ll | 707 +++++++++++++++++-
.../AArch64/vector-extract-last-active.ll | 48 +-
4 files changed, 722 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c19265613c706d..ebfee8f442f365 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15928,11 +15928,20 @@ static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
return getVectorBitwiseReduce(Opcode, HalfVec, VT, DL, DAG);
}
- // Vectors that are less than 64 bits get widened to neatly fit a 64 bit
- // register, so e.g. <4 x i1> gets lowered to <4 x i16>. Sign extending to
- // this element size leads to the best codegen, since e.g. setcc results
- // might need to be truncated otherwise.
- EVT ExtendedVT = MVT::getIntegerVT(std::max(64u / NumElems, 8u));
+ // Results of setcc operations get widened to 128 bits if their input
+ // operands are 128 bits wide and in case of reduce_and and reduce_or have
+ // at least 4 elements, otherwise vectors that are less than 64 bits get
+ // widened to neatly fit a 64 bit register, so e.g. <4 x i1> gets lowered to
+ // either <4 x i16> or <4 x i32>. Sign extending to this element size leads
+ // to the best codegen, since e.g. setcc results might need to be truncated
+ // otherwise.
+ unsigned ExtendedWidth = 64;
+ if ((ScalarOpcode == ISD::XOR || NumElems >= 4) &&
+ Vec.getOpcode() == ISD::SETCC &&
+ Vec.getOperand(0).getValueSizeInBits() >= 128) {
+ ExtendedWidth = 128;
+ }
+ EVT ExtendedVT = MVT::getIntegerVT(std::max(ExtendedWidth / NumElems, 8u));
// any_ext doesn't work with umin/umax, so only use it for uadd.
unsigned ExtendOp =
diff --git a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
index 767ca91a58bb10..f317a7b8083421 100644
--- a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
@@ -12,8 +12,7 @@ define i1 @unordered_floating_point_compare_on_v8f32(<8 x float> %a_vec) {
; CHECK-NEXT: mov w8, #1 // =0x1
; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: umaxv b0, v0.8b
+; CHECK-NEXT: umaxv h0, v0.8h
; CHECK-NEXT: fmov w9, s0
; CHECK-NEXT: bic w0, w8, w9
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
index 58020d28702b2f..55b9162921f88a 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
@@ -15,8 +15,15 @@ declare i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %a)
declare i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %a)
declare i1 @llvm.vector.reduce.or.v32i1(<32 x i1> %a)
-define i32 @reduce_and_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v1:
+declare i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %a)
+declare i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> %a)
+
+define i32 @reduce_and_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: smov w8, v0.b[0]
@@ -29,8 +36,8 @@ define i32 @reduce_and_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_and_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v2:
+define i32 @reduce_and_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i8:
; CHECK: // %bb.0:
; CHECK-NEXT: shl v0.2s, v0.2s, #24
; CHECK-NEXT: sshr v0.2s, v0.2s, #24
@@ -46,8 +53,8 @@ define i32 @reduce_and_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_and_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v4:
+define i32 @reduce_and_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i8:
; CHECK: // %bb.0:
; CHECK-NEXT: shl v0.4h, v0.4h, #8
; CHECK-NEXT: sshr v0.4h, v0.4h, #8
@@ -63,8 +70,8 @@ define i32 @reduce_and_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_and_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v8:
+define i32 @reduce_and_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i8:
; CHECK: // %bb.0:
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: uminv b0, v0.8b
@@ -78,8 +85,8 @@ define i32 @reduce_and_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_and_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v16:
+define i32 @reduce_and_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v16i8:
; CHECK: // %bb.0:
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: uminv b0, v0.16b
@@ -93,8 +100,8 @@ define i32 @reduce_and_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_and_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_and_v32:
+define i32 @reduce_and_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v32i8:
; CHECK: // %bb.0:
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
@@ -109,8 +116,194 @@ define i32 @reduce_and_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v1:
+define i32 @reduce_and_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: smov w8, v0.h[0]
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: uminp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: uminv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: uminv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v16i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: uminv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <16 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: uminp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: uminv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: uminv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: uminp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_and_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_and_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: uminv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: smov w8, v0.b[0]
@@ -123,8 +316,8 @@ define i32 @reduce_or_v1(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v2:
+define i32 @reduce_or_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i8:
; CHECK: // %bb.0:
; CHECK-NEXT: shl v0.2s, v0.2s, #24
; CHECK-NEXT: sshr v0.2s, v0.2s, #24
@@ -140,8 +333,8 @@ define i32 @reduce_or_v2(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v4:
+define i32 @reduce_or_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i8:
; CHECK: // %bb.0:
; CHECK-NEXT: shl v0.4h, v0.4h, #8
; CHECK-NEXT: sshr v0.4h, v0.4h, #8
@@ -157,8 +350,8 @@ define i32 @reduce_or_v4(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v8:
+define i32 @reduce_or_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i8:
; CHECK: // %bb.0:
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: umaxv b0, v0.8b
@@ -172,8 +365,8 @@ define i32 @reduce_or_v8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v16:
+define i32 @reduce_or_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v16i8:
; CHECK: // %bb.0:
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-NEXT: umaxv b0, v0.16b
@@ -187,8 +380,8 @@ define i32 @reduce_or_v16(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
ret i32 %z
}
-define i32 @reduce_or_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
-; CHECK-LABEL: reduce_or_v32:
+define i32 @reduce_or_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v32i8:
; CHECK: // %bb.0:
; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
@@ -202,3 +395,469 @@ define i32 @reduce_or_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
%z = select i1 %y, i32 %a1, i32 %a2
ret i32 %z
}
+
+define i32 @reduce_or_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: smov w8, v0.h[0]
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: umaxv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: umaxv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v16i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: umaxv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <16 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: umaxv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: umaxv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_or_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_or_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: umaxv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: smov w8, v0.b[0]
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v2i8(<2 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-NEXT: sshr v0.2s, v0.2s, #24
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v4i8(<4 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: addv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v8i8(<8 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v16i8(<16 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <16 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v32i8(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <32 x i8> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: smov w8, v0.h[0]
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v2i16(<2 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v4i16(<4 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: addv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v8i16(<8 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v16i16(<16 x i16> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v16i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <16 x i16> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v2i32(<2 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v4i32(<4 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v8i32(<8 x i32> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <8 x i32> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csel w0, w0, w1, lt
+; CHECK-NEXT: ret
+ %x = icmp slt <1 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v2i64(<2 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <2 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
+
+define i32 @reduce_xor_v4i64(<4 x i64> %a0, i32 %a1, i32 %a2) nounwind {
+; CHECK-LABEL: reduce_xor_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tst w8, #0x1
+; CHECK-NEXT: csel w0, w0, w1, ne
+; CHECK-NEXT: ret
+ %x = icmp slt <4 x i64> %a0, zeroinitializer
+ %y = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> %x)
+ %z = select i1 %y, i32 %a1, i32 %a2
+ ret i32 %z
+}
diff --git a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
index c0f1720e1cf8b3..c7d6a3db8eae18 100644
--- a/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/AArch64/vector-extract-last-active.ll
@@ -58,11 +58,11 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
; NEON-FIXED-NEXT: adrp x8, .LCPI1_0
; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d2, [x8, :lo12:.LCPI1_0]
+; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI1_0]
; NEON-FIXED-NEXT: str q0, [sp]
-; NEON-FIXED-NEXT: xtn v1.8b, v1.8h
-; NEON-FIXED-NEXT: and v2.8b, v1.8b, v2.8b
-; NEON-FIXED-NEXT: umaxv b1, v1.8b
+; NEON-FIXED-NEXT: xtn v2.8b, v1.8h
+; NEON-FIXED-NEXT: umaxv h1, v1.8h
+; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
; NEON-FIXED-NEXT: umaxv b2, v2.8b
; NEON-FIXED-NEXT: fmov w8, s2
; NEON-FIXED-NEXT: bfi x9, x8, #1, #3
@@ -78,12 +78,12 @@ define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {
; SVE-FIXED-NEXT: sub sp, sp, #16
; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
; SVE-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h
-; SVE-FIXED-NEXT: index z2.b, #0, #1
+; SVE-FIXED-NEXT: index z3.b, #0, #1
; SVE-FIXED-NEXT: mov x9, sp
; SVE-FIXED-NEXT: str q0, [sp]
-; SVE-FIXED-NEXT: xtn v1.8b, v1.8h
-; SVE-FIXED-NEXT: and v2.8b, v1.8b, v2.8b
-; SVE-FIXED-NEXT: umaxv b1, v1.8b
+; SVE-FIXED-NEXT: xtn v2.8b, v1.8h
+; SVE-FIXED-NEXT: umaxv h1, v1.8h
+; SVE-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
; SVE-FIXED-NEXT: umaxv b2, v2.8b
; SVE-FIXED-NEXT: fmov w8, s2
; SVE-FIXED-NEXT: bfi x9, x8, #1, #3
@@ -106,11 +106,11 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
; NEON-FIXED-NEXT: adrp x8, .LCPI2_0
; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d2, [x8, :lo12:.LCPI2_0]
+; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]
; NEON-FIXED-NEXT: str q0, [sp]
-; NEON-FIXED-NEXT: xtn v1.4h, v1.4s
-; NEON-FIXED-NEXT: and v2.8b, v1.8b, v2.8b
-; NEON-FIXED-NEXT: umaxv h1, v1.4h
+; NEON-FIXED-NEXT: xtn v2.4h, v1.4s
+; NEON-FIXED-NEXT: umaxv s1, v1.4s
+; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
; NEON-FIXED-NEXT: umaxv h2, v2.4h
; NEON-FIXED-NEXT: fmov w8, s2
; NEON-FIXED-NEXT: bfi x9, x8, #2, #2
@@ -126,12 +126,12 @@ define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {
; SVE-FIXED-NEXT: sub sp, sp, #16
; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
; SVE-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
-; SVE-FIXED-NEXT: index z2.h, #0, #1
+; SVE-FIXED-NEXT: index z3.h, #0, #1
; SVE-FIXED-NEXT: mov x9, sp
; SVE-FIXED-NEXT: str q0, [sp]
-; SVE-FIXED-NEXT: xtn v1.4h, v1.4s
-; SVE-FIXED-NEXT: and v2.8b, v1.8b, v2.8b
-; SVE-FIXED-NEXT: umaxv h1, v1.4h
+; SVE-FIXED-NEXT: xtn v2.4h, v1.4s
+; SVE-FIXED-NEXT: umaxv s1, v1.4s
+; SVE-FIXED-NEXT: and v2.8b, v2.8b, v3.8b
; SVE-FIXED-NEXT: umaxv h2, v2.4h
; SVE-FIXED-NEXT: fmov w8, s2
; SVE-FIXED-NEXT: bfi x9, x8, #2, #2
@@ -202,11 +202,11 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
; NEON-FIXED-NEXT: adrp x8, .LCPI4_0
; NEON-FIXED-NEXT: mov x9, sp
-; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI4_0]
+; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI4_0]
; NEON-FIXED-NEXT: str q0, [sp]
-; NEON-FIXED-NEXT: xtn v1.4h, v1.4s
-; NEON-FIXED-NEXT: and v3.8b, v1.8b, v3.8b
-; NEON-FIXED-NEXT: umaxv h1, v1.4h
+; NEON-FIXED-NEXT: xtn v3.4h, v1.4s
+; NEON-FIXED-NEXT: umaxv s1, v1.4s
+; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b
; NEON-FIXED-NEXT: umaxv h3, v3.4h
; NEON-FIXED-NEXT: fmov w8, s3
; NEON-FIXED-NEXT: bfi x9, x8, #2, #2
@@ -222,12 +222,12 @@ define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %pass
; SVE-FIXED-NEXT: sub sp, sp, #16
; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
; SVE-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s
-; SVE-FIXED-NEXT: index z3.h, #0, #1
+; SVE-FIXED-NEXT: index z4.h, #0, #1
; SVE-FIXED-NEXT: mov x9, sp
; SVE-FIXED-NEXT: str q0, [sp]
-; SVE-FIXED-NEXT: xtn v1.4h, v1.4s
-; SVE-FIXED-NEXT: and v3.8b, v1.8b, v3.8b
-; SVE-FIXED-NEXT: umaxv h1, v1.4h
+; SVE-FIXED-NEXT: xtn v3.4h, v1.4s
+; SVE-FIXED-NEXT: umaxv s1, v1.4s
+; SVE-FIXED-NEXT: and v3.8b, v3.8b, v4.8b
; SVE-FIXED-NEXT: umaxv h3, v3.4h
; SVE-FIXED-NEXT: fmov w8, s3
; SVE-FIXED-NEXT: bfi x9, x8, #2, #2
More information about the llvm-commits
mailing list