[llvm] [AArch64] Improve non-SVE popcount for 32bit and 64 bit using udot (PR #95881)
Tim Gymnich via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 18 16:32:35 PDT 2024
https://github.com/tgymnich updated https://github.com/llvm/llvm-project/pull/95881
>From 4ff6376a42e365905a8481322d2534ee492c85e3 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 05:09:17 +0200
Subject: [PATCH 1/9] [AArch64] Improve non-SVE popcount for 32bit and 64 bit
using udot
---
.../Target/AArch64/AArch64ISelLowering.cpp | 20 +++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9f6f66e9e0c70..19aa1b672d7d4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9804,6 +9804,26 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
Val = DAG.getBitcast(VT8Bit, Val);
Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
+ if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16) {
+ EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
+ SDValue Zeros = DAG.getSplatBuildVector(
+ DT, DL, DAG.getConstant(0, DL, DT.getScalarType()));
+ SDValue Ones =
+ DAG.getSplatBuildVector(VT8Bit, DL, DAG.getConstant(1, DL, MVT::i8));
+
+ if (VT == MVT::v2i64) {
+ Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val);
+ Val = DAG.getNode(AArch64ISD::UADDLP, DL, VT, Val);
+ } else if (VT == MVT::v2i32) {
+ Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val);
+ } else if (VT == MVT::v4i32) {
+ Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val);
+ } else {
+ llvm_unreachable("Unexpected type for custom ctpop lowering");
+ }
+
+ return Val;
+ }
// Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
unsigned EltSize = 8;
unsigned NumElts = VT.is64BitVector() ? 8 : 16;
>From 4026b57fcde33c4cef9cbe3db7e9fbafd579f026 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 06:46:42 +0200
Subject: [PATCH 2/9] add tests
---
llvm/test/CodeGen/AArch64/popcount.ll | 295 ++++++++++++++++++++++++++
1 file changed, 295 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index b1231eeac1ea4..c041620fcc104 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -1,5 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefix=NEON
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefix=DOT
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefix=SVE
; Function Attrs: nobuiltin nounwind readonly
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
@@ -12,6 +15,36 @@ define i8 @popcount128(ptr nocapture nonnull readonly %0) {
; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount128:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: ldr d0, [x0]
+; NEON-NEXT: add x8, x0, #8
+; NEON-NEXT: ld1 { v0.d }[1], [x8]
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlv h0, v0.16b
+; NEON-NEXT: fmov w0, s0
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount128:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: ldr d0, [x0]
+; DOT-NEXT: add x8, x0, #8
+; DOT-NEXT: ld1 { v0.d }[1], [x8]
+; DOT-NEXT: cnt v0.16b, v0.16b
+; DOT-NEXT: uaddlv h0, v0.16b
+; DOT-NEXT: fmov w0, s0
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount128:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: ldr d0, [x0]
+; SVE-NEXT: add x8, x0, #8
+; SVE-NEXT: ld1 { v0.d }[1], [x8]
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlv h0, v0.16b
+; SVE-NEXT: fmov w0, s0
+; SVE-NEXT: ret
Entry:
%1 = load i128, ptr %0, align 16
%2 = tail call i128 @llvm.ctpop.i128(i128 %1)
@@ -56,6 +89,57 @@ define i16 @popcount256(ptr nocapture nonnull readonly %0) {
; CHECK-NEXT: adds x8, x8, x9
; CHECK-NEXT: mov w0, w8
; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount256:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: ldr d0, [x0, #16]
+; NEON-NEXT: ldr d1, [x0]
+; NEON-NEXT: add x8, x0, #8
+; NEON-NEXT: add x9, x0, #24
+; NEON-NEXT: ld1 { v0.d }[1], [x9]
+; NEON-NEXT: ld1 { v1.d }[1], [x8]
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: cnt v1.16b, v1.16b
+; NEON-NEXT: uaddlv h0, v0.16b
+; NEON-NEXT: uaddlv h1, v1.16b
+; NEON-NEXT: fmov w8, s0
+; NEON-NEXT: fmov w9, s1
+; NEON-NEXT: add w0, w9, w8
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount256:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: ldr d0, [x0, #16]
+; DOT-NEXT: ldr d1, [x0]
+; DOT-NEXT: add x8, x0, #8
+; DOT-NEXT: add x9, x0, #24
+; DOT-NEXT: ld1 { v0.d }[1], [x9]
+; DOT-NEXT: ld1 { v1.d }[1], [x8]
+; DOT-NEXT: cnt v0.16b, v0.16b
+; DOT-NEXT: cnt v1.16b, v1.16b
+; DOT-NEXT: uaddlv h0, v0.16b
+; DOT-NEXT: uaddlv h1, v1.16b
+; DOT-NEXT: fmov w8, s0
+; DOT-NEXT: fmov w9, s1
+; DOT-NEXT: add w0, w9, w8
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount256:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: ldr d0, [x0, #16]
+; SVE-NEXT: ldr d1, [x0]
+; SVE-NEXT: add x8, x0, #8
+; SVE-NEXT: add x9, x0, #24
+; SVE-NEXT: ld1 { v0.d }[1], [x9]
+; SVE-NEXT: ld1 { v1.d }[1], [x8]
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: cnt v1.16b, v1.16b
+; SVE-NEXT: uaddlv h0, v0.16b
+; SVE-NEXT: uaddlv h1, v1.16b
+; SVE-NEXT: fmov w8, s0
+; SVE-NEXT: fmov w9, s1
+; SVE-NEXT: add w0, w9, w8
+; SVE-NEXT: ret
Entry:
%1 = load i256, ptr %0, align 16
%2 = tail call i256 @llvm.ctpop.i256(i256 %1)
@@ -83,9 +167,220 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) {
; CHECK-NEXT: // kill: def $x8 killed $w8
; CHECK-NEXT: bfi x0, x8, #32, #32
; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount1x128:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: fmov d1, x0
+; NEON-NEXT: movi v0.2d, #0000000000000000
+; NEON-NEXT: mov v1.d[1], x1
+; NEON-NEXT: cnt v1.16b, v1.16b
+; NEON-NEXT: uaddlv h1, v1.16b
+; NEON-NEXT: mov v0.s[0], v1.s[0]
+; NEON-NEXT: mov x1, v0.d[1]
+; NEON-NEXT: fmov x0, d0
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount1x128:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: fmov d1, x0
+; DOT-NEXT: movi v0.2d, #0000000000000000
+; DOT-NEXT: mov v1.d[1], x1
+; DOT-NEXT: cnt v1.16b, v1.16b
+; DOT-NEXT: uaddlv h1, v1.16b
+; DOT-NEXT: mov v0.s[0], v1.s[0]
+; DOT-NEXT: mov x1, v0.d[1]
+; DOT-NEXT: fmov x0, d0
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount1x128:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: fmov d1, x0
+; SVE-NEXT: movi v0.2d, #0000000000000000
+; SVE-NEXT: mov v1.d[1], x1
+; SVE-NEXT: cnt v1.16b, v1.16b
+; SVE-NEXT: uaddlv h1, v1.16b
+; SVE-NEXT: mov v0.s[0], v1.s[0]
+; SVE-NEXT: mov x1, v0.d[1]
+; SVE-NEXT: fmov x0, d0
+; SVE-NEXT: ret
Entry:
%1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0)
ret <1 x i128> %1
}
declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)
+
+define <2 x i64> @popcount2x64(<2 x i64> %0) {
+; CHECK-LABEL: popcount2x64:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: uaddlp v0.4s, v0.8h
+; CHECK-NEXT: uaddlp v0.2d, v0.4s
+; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount2x64:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount2x64:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.16b, #1
+; DOT-NEXT: cnt v0.16b, v0.16b
+; DOT-NEXT: movi v2.2d, #0000000000000000
+; DOT-NEXT: udot v2.4s, v1.16b, v0.16b
+; DOT-NEXT: uaddlp v0.2d, v2.4s
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount2x64:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlp v0.8h, v0.16b
+; SVE-NEXT: uaddlp v0.4s, v0.8h
+; SVE-NEXT: uaddlp v0.2d, v0.4s
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <2 x i64> @llvm.ctpop.v2.i64(<2 x i64> %0)
+ ret <2 x i64> %1
+}
+
+declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>)
+
+define <4 x i32> @popcount4x32(<4 x i32> %0) {
+; CHECK-LABEL: popcount4x32:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: uaddlp v0.4s, v0.8h
+; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount4x32:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount4x32:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.16b, #1
+; DOT-NEXT: cnt v2.16b, v0.16b
+; DOT-NEXT: movi v0.2d, #0000000000000000
+; DOT-NEXT: udot v0.4s, v1.16b, v2.16b
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount4x32:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlp v0.8h, v0.16b
+; SVE-NEXT: uaddlp v0.4s, v0.8h
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <4 x i32> @llvm.ctpop.v4.i32(<4 x i32> %0)
+ ret <4 x i32> %1
+}
+
+declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>)
+
+define <2 x i32> @popcount2x32(<2 x i32> %0) {
+; CHECK-LABEL: popcount2x32:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: uaddlp v0.2s, v0.4h
+; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount2x32:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount2x32:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.2d, #0000000000000000
+; DOT-NEXT: cnt v0.8b, v0.8b
+; DOT-NEXT: movi v2.8b, #1
+; DOT-NEXT: udot v1.2s, v2.8b, v0.8b
+; DOT-NEXT: fmov d0, d1
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount2x32:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: uaddlp v0.4h, v0.8b
+; SVE-NEXT: uaddlp v0.2s, v0.4h
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <2 x i32> @llvm.ctpop.v2.i32(<2 x i32> %0)
+ ret <2 x i32> %1
+}
+
+declare <2 x i32> @llvm.ctpop.v2.i32(<2 x i32>)
+
+define <8 x i16> @popcount8x16(<8 x i16> %0) {
+; CHECK-LABEL: popcount8x16:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount8x16:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount8x16:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: cnt v0.16b, v0.16b
+; DOT-NEXT: uaddlp v0.8h, v0.16b
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount8x16:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlp v0.8h, v0.16b
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0)
+ ret <8 x i16> %1
+}
+
+declare <8 x i16> @llvm.ctpop.v8.i16(<8 x i16>)
+
+define <4 x i16> @popcount4x16(<4 x i16> %0) {
+; CHECK-LABEL: popcount4x16:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: ret
+;
+; NEON-LABEL: popcount4x16:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: ret
+;
+; DOT-LABEL: popcount4x16:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: cnt v0.8b, v0.8b
+; DOT-NEXT: uaddlp v0.4h, v0.8b
+; DOT-NEXT: ret
+;
+; SVE-LABEL: popcount4x16:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: uaddlp v0.4h, v0.8b
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0)
+ ret <4 x i16> %1
+}
+
+declare <4 x i16> @llvm.ctpop.v4.i16(<4 x i16>)
>From bc35cfdf090daecee59a05e3f8fb11e6edb2f83e Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 13:53:00 +0200
Subject: [PATCH 3/9] combine check prefixes
---
llvm/test/CodeGen/AArch64/popcount.ll | 124 +-------------------------
1 file changed, 3 insertions(+), 121 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index c041620fcc104..5329d619e58a9 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -1,21 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s
-; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefix=NEON
-; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefix=DOT
-; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefix=SVE
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE
; Function Attrs: nobuiltin nounwind readonly
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
-; CHECK-LABEL: popcount128:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: cnt v0.16b, v0.16b
-; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount128:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: ldr d0, [x0]
@@ -57,39 +47,6 @@ declare i128 @llvm.ctpop.i128(i128)
; Function Attrs: nobuiltin nounwind readonly
define i16 @popcount256(ptr nocapture nonnull readonly %0) {
-; CHECK-LABEL: popcount256:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: ldr x11, [x0]
-; CHECK-NEXT: ldr x10, [x0, #8]
-; CHECK-NEXT: ldr x9, [x0, #16]
-; CHECK-NEXT: ldr x8, [x0, #24]
-; CHECK-NEXT: // implicit-def: $q1
-; CHECK-NEXT: mov v1.d[0], x11
-; CHECK-NEXT: mov v1.d[1], x10
-; CHECK-NEXT: // implicit-def: $q0
-; CHECK-NEXT: mov v0.d[0], x9
-; CHECK-NEXT: mov v0.d[1], x8
-; CHECK-NEXT: cnt v1.16b, v1.16b
-; CHECK-NEXT: uaddlv h1, v1.16b
-; CHECK-NEXT: // kill: def $q1 killed $h1
-; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
-; CHECK-NEXT: fmov w0, s1
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: mov w9, w0
-; CHECK-NEXT: mov w8, w10
-; CHECK-NEXT: bfi x9, x8, #32, #32
-; CHECK-NEXT: cnt v0.16b, v0.16b
-; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: mov w8, w0
-; CHECK-NEXT: // kill: def $x10 killed $w10
-; CHECK-NEXT: bfi x8, x10, #32, #32
-; CHECK-NEXT: adds x8, x8, x9
-; CHECK-NEXT: mov w0, w8
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount256:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: ldr d0, [x0, #16]
@@ -151,23 +108,6 @@ Entry:
declare i256 @llvm.ctpop.i256(i256)
define <1 x i128> @popcount1x128(<1 x i128> %0) {
-; CHECK-LABEL: popcount1x128:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: // implicit-def: $q0
-; CHECK-NEXT: mov v0.d[0], x0
-; CHECK-NEXT: mov v0.d[1], x1
-; CHECK-NEXT: cnt v0.16b, v0.16b
-; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: mov x1, xzr
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: // kill: def $x0 killed $w0
-; CHECK-NEXT: // kill: def $x8 killed $w8
-; CHECK-NEXT: bfi x0, x8, #32, #32
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount1x128:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: fmov d1, x0
@@ -211,14 +151,6 @@ Entry:
declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)
define <2 x i64> @popcount2x64(<2 x i64> %0) {
-; CHECK-LABEL: popcount2x64:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: cnt v0.16b, v0.16b
-; CHECK-NEXT: uaddlp v0.8h, v0.16b
-; CHECK-NEXT: uaddlp v0.4s, v0.8h
-; CHECK-NEXT: uaddlp v0.2d, v0.4s
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount2x64:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.16b, v0.16b
@@ -251,13 +183,6 @@ Entry:
declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>)
define <4 x i32> @popcount4x32(<4 x i32> %0) {
-; CHECK-LABEL: popcount4x32:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: cnt v0.16b, v0.16b
-; CHECK-NEXT: uaddlp v0.8h, v0.16b
-; CHECK-NEXT: uaddlp v0.4s, v0.8h
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount4x32:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.16b, v0.16b
@@ -287,13 +212,6 @@ Entry:
declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>)
define <2 x i32> @popcount2x32(<2 x i32> %0) {
-; CHECK-LABEL: popcount2x32:
-; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: cnt v0.8b, v0.8b
-; CHECK-NEXT: uaddlp v0.4h, v0.8b
-; CHECK-NEXT: uaddlp v0.2s, v0.4h
-; CHECK-NEXT: ret
-;
; NEON-LABEL: popcount2x32:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.8b, v0.8b
@@ -329,24 +247,6 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) {
; CHECK-NEXT: cnt v0.16b, v0.16b
; CHECK-NEXT: uaddlp v0.8h, v0.16b
; CHECK-NEXT: ret
-;
-; NEON-LABEL: popcount8x16:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: cnt v0.16b, v0.16b
-; NEON-NEXT: uaddlp v0.8h, v0.16b
-; NEON-NEXT: ret
-;
-; DOT-LABEL: popcount8x16:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: cnt v0.16b, v0.16b
-; DOT-NEXT: uaddlp v0.8h, v0.16b
-; DOT-NEXT: ret
-;
-; SVE-LABEL: popcount8x16:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: cnt v0.16b, v0.16b
-; SVE-NEXT: uaddlp v0.8h, v0.16b
-; SVE-NEXT: ret
Entry:
%1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0)
ret <8 x i16> %1
@@ -360,24 +260,6 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) {
; CHECK-NEXT: cnt v0.8b, v0.8b
; CHECK-NEXT: uaddlp v0.4h, v0.8b
; CHECK-NEXT: ret
-;
-; NEON-LABEL: popcount4x16:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: cnt v0.8b, v0.8b
-; NEON-NEXT: uaddlp v0.4h, v0.8b
-; NEON-NEXT: ret
-;
-; DOT-LABEL: popcount4x16:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: cnt v0.8b, v0.8b
-; DOT-NEXT: uaddlp v0.4h, v0.8b
-; DOT-NEXT: ret
-;
-; SVE-LABEL: popcount4x16:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: cnt v0.8b, v0.8b
-; SVE-NEXT: uaddlp v0.4h, v0.8b
-; SVE-NEXT: ret
Entry:
%1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0)
ret <4 x i16> %1
>From 68eeaccb67af8e91a4c298d0ca8b7fa106e514c3 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 15:09:41 +0200
Subject: [PATCH 4/9] exclude v1i64
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 19aa1b672d7d4..8bc219727ca1f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9804,7 +9804,8 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
Val = DAG.getBitcast(VT8Bit, Val);
Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
- if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16) {
+ if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16 &&
+ VT.getVectorNumElements() >= 2) {
EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
SDValue Zeros = DAG.getSplatBuildVector(
DT, DL, DAG.getConstant(0, DL, DT.getScalarType()));
@@ -9824,6 +9825,7 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
return Val;
}
+
// Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
unsigned EltSize = 8;
unsigned NumElts = VT.is64BitVector() ? 8 : 16;
>From 841d1666e3e03ff6ba21f40dc9100c19712122d2 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 15:10:23 +0200
Subject: [PATCH 5/9] use getConstant
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8bc219727ca1f..bf9ebaa638274 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9807,10 +9807,8 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16 &&
VT.getVectorNumElements() >= 2) {
EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
- SDValue Zeros = DAG.getSplatBuildVector(
- DT, DL, DAG.getConstant(0, DL, DT.getScalarType()));
- SDValue Ones =
- DAG.getSplatBuildVector(VT8Bit, DL, DAG.getConstant(1, DL, MVT::i8));
+ SDValue Zeros = DAG.getConstant(0, DL, DT);
+ SDValue Ones = DAG.getConstant(1, DL, VT8Bit);
if (VT == MVT::v2i64) {
Val = DAG.getNode(AArch64ISD::UDOT, DL, DT, Zeros, Ones, Val);
>From f68ef32d377190e5ade3b39912161d201d5e9ac6 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 19:56:15 +0200
Subject: [PATCH 6/9] fix check prefix
---
llvm/test/CodeGen/AArch64/popcount.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index 5329d619e58a9..c0cd35258ee3a 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO0
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE
>From 0692f850cd9060fcc0d7de8a5ee36beb2de19c9b Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 19:56:54 +0200
Subject: [PATCH 7/9] fix intrinsic names
---
llvm/test/CodeGen/AArch64/popcount.ll | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index c0cd35258ee3a..d50c50d7aee29 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -144,11 +144,11 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) {
; SVE-NEXT: fmov x0, d0
; SVE-NEXT: ret
Entry:
- %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0)
+ %1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)
ret <1 x i128> %1
}
-declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)
+declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)
define <2 x i64> @popcount2x64(<2 x i64> %0) {
; NEON-LABEL: popcount2x64:
@@ -176,11 +176,11 @@ define <2 x i64> @popcount2x64(<2 x i64> %0) {
; SVE-NEXT: uaddlp v0.2d, v0.4s
; SVE-NEXT: ret
Entry:
- %1 = tail call <2 x i64> @llvm.ctpop.v2.i64(<2 x i64> %0)
+ %1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
ret <2 x i64> %1
}
-declare <2 x i64> @llvm.ctpop.v2.i64(<2 x i64>)
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
define <4 x i32> @popcount4x32(<4 x i32> %0) {
; NEON-LABEL: popcount4x32:
@@ -205,11 +205,11 @@ define <4 x i32> @popcount4x32(<4 x i32> %0) {
; SVE-NEXT: uaddlp v0.4s, v0.8h
; SVE-NEXT: ret
Entry:
- %1 = tail call <4 x i32> @llvm.ctpop.v4.i32(<4 x i32> %0)
+ %1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
ret <4 x i32> %1
}
-declare <4 x i32> @llvm.ctpop.v4.i32(<4 x i32>)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
define <2 x i32> @popcount2x32(<2 x i32> %0) {
; NEON-LABEL: popcount2x32:
@@ -235,11 +235,11 @@ define <2 x i32> @popcount2x32(<2 x i32> %0) {
; SVE-NEXT: uaddlp v0.2s, v0.4h
; SVE-NEXT: ret
Entry:
- %1 = tail call <2 x i32> @llvm.ctpop.v2.i32(<2 x i32> %0)
+ %1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)
ret <2 x i32> %1
}
-declare <2 x i32> @llvm.ctpop.v2.i32(<2 x i32>)
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
define <8 x i16> @popcount8x16(<8 x i16> %0) {
; CHECK-LABEL: popcount8x16:
@@ -248,11 +248,11 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) {
; CHECK-NEXT: uaddlp v0.8h, v0.16b
; CHECK-NEXT: ret
Entry:
- %1 = tail call <8 x i16> @llvm.ctpop.v8.i16(<8 x i16> %0)
+ %1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0)
ret <8 x i16> %1
}
-declare <8 x i16> @llvm.ctpop.v8.i16(<8 x i16>)
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
define <4 x i16> @popcount4x16(<4 x i16> %0) {
; CHECK-LABEL: popcount4x16:
@@ -261,8 +261,8 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) {
; CHECK-NEXT: uaddlp v0.4h, v0.8b
; CHECK-NEXT: ret
Entry:
- %1 = tail call <4 x i16> @llvm.ctpop.v4.i16(<4 x i16> %0)
+ %1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0)
ret <4 x i16> %1
}
-declare <4 x i16> @llvm.ctpop.v4.i16(<4 x i16>)
+declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
>From 39e82eb1f75ceda94f6b96ce806f07d72fdee81b Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 18 Jun 2024 20:05:54 +0200
Subject: [PATCH 8/9] update tests
---
llvm/test/CodeGen/AArch64/popcount.ll | 238 ++++++++++++++------------
1 file changed, 127 insertions(+), 111 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index d50c50d7aee29..9a0b2299e8d24 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -6,35 +6,25 @@
; Function Attrs: nobuiltin nounwind readonly
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
-; NEON-LABEL: popcount128:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: ldr d0, [x0]
-; NEON-NEXT: add x8, x0, #8
-; NEON-NEXT: ld1 { v0.d }[1], [x8]
-; NEON-NEXT: cnt v0.16b, v0.16b
-; NEON-NEXT: uaddlv h0, v0.16b
-; NEON-NEXT: fmov w0, s0
-; NEON-NEXT: ret
+; CHECKO0-LABEL: popcount128:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: ldr q0, [x0]
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: ret
;
-; DOT-LABEL: popcount128:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: ldr d0, [x0]
-; DOT-NEXT: add x8, x0, #8
-; DOT-NEXT: ld1 { v0.d }[1], [x8]
-; DOT-NEXT: cnt v0.16b, v0.16b
-; DOT-NEXT: uaddlv h0, v0.16b
-; DOT-NEXT: fmov w0, s0
-; DOT-NEXT: ret
-;
-; SVE-LABEL: popcount128:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: ldr d0, [x0]
-; SVE-NEXT: add x8, x0, #8
-; SVE-NEXT: ld1 { v0.d }[1], [x8]
-; SVE-NEXT: cnt v0.16b, v0.16b
-; SVE-NEXT: uaddlv h0, v0.16b
-; SVE-NEXT: fmov w0, s0
-; SVE-NEXT: ret
+; CHECK-LABEL: popcount128:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: add x8, x0, #8
+; CHECK-NEXT: ld1 { v0.d }[1], [x8]
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlv h0, v0.16b
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
Entry:
%1 = load i128, ptr %0, align 16
%2 = tail call i128 @llvm.ctpop.i128(i128 %1)
@@ -47,56 +37,55 @@ declare i128 @llvm.ctpop.i128(i128)
; Function Attrs: nobuiltin nounwind readonly
define i16 @popcount256(ptr nocapture nonnull readonly %0) {
-; NEON-LABEL: popcount256:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: ldr d0, [x0, #16]
-; NEON-NEXT: ldr d1, [x0]
-; NEON-NEXT: add x8, x0, #8
-; NEON-NEXT: add x9, x0, #24
-; NEON-NEXT: ld1 { v0.d }[1], [x9]
-; NEON-NEXT: ld1 { v1.d }[1], [x8]
-; NEON-NEXT: cnt v0.16b, v0.16b
-; NEON-NEXT: cnt v1.16b, v1.16b
-; NEON-NEXT: uaddlv h0, v0.16b
-; NEON-NEXT: uaddlv h1, v1.16b
-; NEON-NEXT: fmov w8, s0
-; NEON-NEXT: fmov w9, s1
-; NEON-NEXT: add w0, w9, w8
-; NEON-NEXT: ret
+; CHECKO0-LABEL: popcount256:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: ldr x11, [x0]
+; CHECKO0-NEXT: ldr x10, [x0, #8]
+; CHECKO0-NEXT: ldr x9, [x0, #16]
+; CHECKO0-NEXT: ldr x8, [x0, #24]
+; CHECKO0-NEXT: // implicit-def: $q1
+; CHECKO0-NEXT: mov v1.d[0], x11
+; CHECKO0-NEXT: mov v1.d[1], x10
+; CHECKO0-NEXT: // implicit-def: $q0
+; CHECKO0-NEXT: mov v0.d[0], x9
+; CHECKO0-NEXT: mov v0.d[1], x8
+; CHECKO0-NEXT: cnt v1.16b, v1.16b
+; CHECKO0-NEXT: uaddlv h1, v1.16b
+; CHECKO0-NEXT: // kill: def $q1 killed $h1
+; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1
+; CHECKO0-NEXT: fmov w0, s1
+; CHECKO0-NEXT: mov w10, wzr
+; CHECKO0-NEXT: mov w9, w0
+; CHECKO0-NEXT: mov w8, w10
+; CHECKO0-NEXT: bfi x9, x8, #32, #32
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: mov w8, w0
+; CHECKO0-NEXT: // kill: def $x10 killed $w10
+; CHECKO0-NEXT: bfi x8, x10, #32, #32
+; CHECKO0-NEXT: adds x8, x8, x9
+; CHECKO0-NEXT: mov w0, w8
+; CHECKO0-NEXT: ret
;
-; DOT-LABEL: popcount256:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: ldr d0, [x0, #16]
-; DOT-NEXT: ldr d1, [x0]
-; DOT-NEXT: add x8, x0, #8
-; DOT-NEXT: add x9, x0, #24
-; DOT-NEXT: ld1 { v0.d }[1], [x9]
-; DOT-NEXT: ld1 { v1.d }[1], [x8]
-; DOT-NEXT: cnt v0.16b, v0.16b
-; DOT-NEXT: cnt v1.16b, v1.16b
-; DOT-NEXT: uaddlv h0, v0.16b
-; DOT-NEXT: uaddlv h1, v1.16b
-; DOT-NEXT: fmov w8, s0
-; DOT-NEXT: fmov w9, s1
-; DOT-NEXT: add w0, w9, w8
-; DOT-NEXT: ret
-;
-; SVE-LABEL: popcount256:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: ldr d0, [x0, #16]
-; SVE-NEXT: ldr d1, [x0]
-; SVE-NEXT: add x8, x0, #8
-; SVE-NEXT: add x9, x0, #24
-; SVE-NEXT: ld1 { v0.d }[1], [x9]
-; SVE-NEXT: ld1 { v1.d }[1], [x8]
-; SVE-NEXT: cnt v0.16b, v0.16b
-; SVE-NEXT: cnt v1.16b, v1.16b
-; SVE-NEXT: uaddlv h0, v0.16b
-; SVE-NEXT: uaddlv h1, v1.16b
-; SVE-NEXT: fmov w8, s0
-; SVE-NEXT: fmov w9, s1
-; SVE-NEXT: add w0, w9, w8
-; SVE-NEXT: ret
+; CHECK-LABEL: popcount256:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: ldr d0, [x0, #16]
+; CHECK-NEXT: ldr d1, [x0]
+; CHECK-NEXT: add x8, x0, #8
+; CHECK-NEXT: add x9, x0, #24
+; CHECK-NEXT: ld1 { v0.d }[1], [x9]
+; CHECK-NEXT: ld1 { v1.d }[1], [x8]
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: cnt v1.16b, v1.16b
+; CHECK-NEXT: uaddlv h0, v0.16b
+; CHECK-NEXT: uaddlv h1, v1.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w0, w9, w8
+; CHECK-NEXT: ret
Entry:
%1 = load i256, ptr %0, align 16
%2 = tail call i256 @llvm.ctpop.i256(i256 %1)
@@ -108,41 +97,34 @@ Entry:
declare i256 @llvm.ctpop.i256(i256)
define <1 x i128> @popcount1x128(<1 x i128> %0) {
-; NEON-LABEL: popcount1x128:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: fmov d1, x0
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: mov v1.d[1], x1
-; NEON-NEXT: cnt v1.16b, v1.16b
-; NEON-NEXT: uaddlv h1, v1.16b
-; NEON-NEXT: mov v0.s[0], v1.s[0]
-; NEON-NEXT: mov x1, v0.d[1]
-; NEON-NEXT: fmov x0, d0
-; NEON-NEXT: ret
-;
-; DOT-LABEL: popcount1x128:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: fmov d1, x0
-; DOT-NEXT: movi v0.2d, #0000000000000000
-; DOT-NEXT: mov v1.d[1], x1
-; DOT-NEXT: cnt v1.16b, v1.16b
-; DOT-NEXT: uaddlv h1, v1.16b
-; DOT-NEXT: mov v0.s[0], v1.s[0]
-; DOT-NEXT: mov x1, v0.d[1]
-; DOT-NEXT: fmov x0, d0
-; DOT-NEXT: ret
+; CHECKO0-LABEL: popcount1x128:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: // implicit-def: $q0
+; CHECKO0-NEXT: mov v0.d[0], x0
+; CHECKO0-NEXT: mov v0.d[1], x1
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: mov x1, xzr
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: mov w8, wzr
+; CHECKO0-NEXT: // kill: def $x0 killed $w0
+; CHECKO0-NEXT: // kill: def $x8 killed $w8
+; CHECKO0-NEXT: bfi x0, x8, #32, #32
+; CHECKO0-NEXT: ret
;
-; SVE-LABEL: popcount1x128:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: fmov d1, x0
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: mov v1.d[1], x1
-; SVE-NEXT: cnt v1.16b, v1.16b
-; SVE-NEXT: uaddlv h1, v1.16b
-; SVE-NEXT: mov v0.s[0], v1.s[0]
-; SVE-NEXT: mov x1, v0.d[1]
-; SVE-NEXT: fmov x0, d0
-; SVE-NEXT: ret
+; CHECK-LABEL: popcount1x128:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: fmov d1, x0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov v1.d[1], x1
+; CHECK-NEXT: cnt v1.16b, v1.16b
+; CHECK-NEXT: uaddlv h1, v1.16b
+; CHECK-NEXT: mov v0.s[0], v1.s[0]
+; CHECK-NEXT: mov x1, v0.d[1]
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
Entry:
%1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)
ret <1 x i128> %1
@@ -151,6 +133,14 @@ Entry:
declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)
define <2 x i64> @popcount2x64(<2 x i64> %0) {
+; CHECKO0-LABEL: popcount2x64:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
+; CHECKO0-NEXT: uaddlp v0.2d, v0.4s
+; CHECKO0-NEXT: ret
+;
; NEON-LABEL: popcount2x64:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.16b, v0.16b
@@ -183,6 +173,13 @@ Entry:
declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
define <4 x i32> @popcount4x32(<4 x i32> %0) {
+; CHECKO0-LABEL: popcount4x32:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
+; CHECKO0-NEXT: ret
+;
; NEON-LABEL: popcount4x32:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.16b, v0.16b
@@ -212,6 +209,13 @@ Entry:
declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
define <2 x i32> @popcount2x32(<2 x i32> %0) {
+; CHECKO0-LABEL: popcount2x32:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
+; CHECKO0-NEXT: uaddlp v0.2s, v0.4h
+; CHECKO0-NEXT: ret
+;
; NEON-LABEL: popcount2x32:
; NEON: // %bb.0: // %Entry
; NEON-NEXT: cnt v0.8b, v0.8b
@@ -242,6 +246,12 @@ Entry:
declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
define <8 x i16> @popcount8x16(<8 x i16> %0) {
+; CHECKO0-LABEL: popcount8x16:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: ret
+;
; CHECK-LABEL: popcount8x16:
; CHECK: // %bb.0: // %Entry
; CHECK-NEXT: cnt v0.16b, v0.16b
@@ -255,6 +265,12 @@ Entry:
declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
define <4 x i16> @popcount4x16(<4 x i16> %0) {
+; CHECKO0-LABEL: popcount4x16:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
+; CHECKO0-NEXT: ret
+;
; CHECK-LABEL: popcount4x16:
; CHECK: // %bb.0: // %Entry
; CHECK-NEXT: cnt v0.8b, v0.8b
>From 4fc15a61ef6c82d86565bb5457ea2f4c235a931e Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Wed, 19 Jun 2024 01:32:18 +0200
Subject: [PATCH 9/9] add v1i64 test
---
llvm/test/CodeGen/AArch64/popcount.ll | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index 9a0b2299e8d24..f60d0db43c4a9 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -172,6 +172,33 @@ Entry:
declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+define <1 x i64> @popcount1x64(<1 x i64> %0) {
+; CHECKO0-LABEL: popcount1x64:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: fmov x0, d0
+; CHECKO0-NEXT: fmov d0, x0
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlv h0, v0.8b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: mov w8, v0.s[0]
+; CHECKO0-NEXT: // kill: def $x8 killed $w8
+; CHECKO0-NEXT: fmov d0, x8
+; CHECKO0-NEXT: ret
+;
+; CHECK-LABEL: popcount1x64:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: uaddlp v0.2s, v0.4h
+; CHECK-NEXT: uaddlp v0.1d, v0.2s
+; CHECK-NEXT: ret
+Entry:
+ %1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0)
+ ret <1 x i64> %1
+}
+
+declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
+
define <4 x i32> @popcount4x32(<4 x i32> %0) {
; CHECKO0-LABEL: popcount4x32:
; CHECKO0: // %bb.0: // %Entry
More information about the llvm-commits
mailing list