[llvm] [AArch64][GlobalISel] Improve non-SVE popcount for 32bit and 64 bit using udot (PR #96409)
Tim Gymnich via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 25 05:46:43 PDT 2024
https://github.com/tgymnich updated https://github.com/llvm/llvm-project/pull/96409
>From f23a13c4dbaa79762861e06fab645bd26503df2a Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Sun, 23 Jun 2024 00:38:17 +0200
Subject: [PATCH 1/4] [AArch64][GlobalISel] Improve non-SVE popcount for 32bit
and 64 bit using udot
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 26 +++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index fef0b722efe45..8f127c418a36b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -14,6 +14,7 @@
#include "AArch64LegalizerInfo.h"
#include "AArch64RegisterBankInfo.h"
#include "AArch64Subtarget.h"
+#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
@@ -1904,6 +1905,31 @@ bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
auto CTPOP = MIRBuilder.buildCTPOP(VTy, Val);
// Sum across lanes.
+
+ if (ST->hasDotProd() && Ty.isVector() && Ty.getNumElements() >= 2 &&
+ Ty.getScalarSizeInBits() != 16) {
+ LLT Dt = Ty == LLT::fixed_vector(2, 64) ? LLT::fixed_vector(4, 32) : Ty;
+ auto Zeros = MIRBuilder.buildConstant(Dt, 0);
+ auto Ones = MIRBuilder.buildConstant(VTy, 1);
+ MachineInstrBuilder SUM;
+
+ if (Ty == LLT::fixed_vector(2, 64)) {
+ auto UDOT =
+ MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
+ SUM = MIRBuilder.buildInstr(AArch64::G_UADDLP, {Ty}, {UDOT});
+ } else if (Ty == LLT::fixed_vector(4, 32)) {
+ SUM = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
+ } else if (Ty == LLT::fixed_vector(2, 32)) {
+ SUM = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
+ } else {
+ llvm_unreachable("unexpected vector shape");
+ }
+
+ SUM->getOperand(0).setReg(Dst);
+ MI.eraseFromParent();
+ return true;
+ }
+
Register HSum = CTPOP.getReg(0);
unsigned Opc;
SmallVector<LLT> HAddTys;
>From 76de62c9013b9e214508637193b5316e0b6c1233 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Mon, 24 Jun 2024 14:49:51 +0200
Subject: [PATCH 2/4] add tests
---
llvm/test/CodeGen/AArch64/popcount.ll | 466 +++++++++++++++++++++++---
1 file changed, 426 insertions(+), 40 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index b1231eeac1ea4..a5b32e8948f92 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -1,15 +1,37 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO0
+; RUN: llc < %s -O0 -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefixes=GISEL
+; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON-GISEL
+; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT-GISEL
+; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE-GISEL
; Function Attrs: nobuiltin nounwind readonly
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
+; CHECKO0-LABEL: popcount128:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: ldr q0, [x0]
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount128:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: ldr q0, [x0]
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlv h0, v0.16b
+; GISEL-NEXT: // kill: def $q0 killed $h0
+; GISEL-NEXT: // kill: def $s0 killed $s0 killed $q0
+; GISEL-NEXT: fmov w0, s0
+; GISEL-NEXT: ret
+;
; CHECK-LABEL: popcount128:
; CHECK: // %bb.0: // %Entry
; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: cnt v0.16b, v0.16b
; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
Entry:
@@ -24,37 +46,88 @@ declare i128 @llvm.ctpop.i128(i128)
; Function Attrs: nobuiltin nounwind readonly
define i16 @popcount256(ptr nocapture nonnull readonly %0) {
+; CHECKO0-LABEL: popcount256:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: ldr x11, [x0]
+; CHECKO0-NEXT: ldr x10, [x0, #8]
+; CHECKO0-NEXT: ldr x9, [x0, #16]
+; CHECKO0-NEXT: ldr x8, [x0, #24]
+; CHECKO0-NEXT: // implicit-def: $q1
+; CHECKO0-NEXT: mov v1.d[0], x11
+; CHECKO0-NEXT: mov v1.d[1], x10
+; CHECKO0-NEXT: // implicit-def: $q0
+; CHECKO0-NEXT: mov v0.d[0], x9
+; CHECKO0-NEXT: mov v0.d[1], x8
+; CHECKO0-NEXT: cnt v1.16b, v1.16b
+; CHECKO0-NEXT: uaddlv h1, v1.16b
+; CHECKO0-NEXT: // kill: def $q1 killed $h1
+; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1
+; CHECKO0-NEXT: fmov w0, s1
+; CHECKO0-NEXT: mov w10, wzr
+; CHECKO0-NEXT: mov w9, w0
+; CHECKO0-NEXT: mov w8, w10
+; CHECKO0-NEXT: bfi x9, x8, #32, #32
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: mov w8, w0
+; CHECKO0-NEXT: // kill: def $x10 killed $w10
+; CHECKO0-NEXT: bfi x8, x10, #32, #32
+; CHECKO0-NEXT: adds x8, x8, x9
+; CHECKO0-NEXT: mov w0, w8
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount256:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: ldr x11, [x0]
+; GISEL-NEXT: ldr x10, [x0, #8]
+; GISEL-NEXT: ldr x9, [x0, #16]
+; GISEL-NEXT: ldr x8, [x0, #24]
+; GISEL-NEXT: // implicit-def: $q1
+; GISEL-NEXT: mov v1.d[0], x11
+; GISEL-NEXT: mov v1.d[1], x10
+; GISEL-NEXT: // implicit-def: $q0
+; GISEL-NEXT: mov v0.d[0], x9
+; GISEL-NEXT: mov v0.d[1], x8
+; GISEL-NEXT: cnt v1.16b, v1.16b
+; GISEL-NEXT: uaddlv h1, v1.16b
+; GISEL-NEXT: // kill: def $q1 killed $h1
+; GISEL-NEXT: // kill: def $s1 killed $s1 killed $q1
+; GISEL-NEXT: fmov w0, s1
+; GISEL-NEXT: mov w10, wzr
+; GISEL-NEXT: mov w9, w0
+; GISEL-NEXT: mov w8, w10
+; GISEL-NEXT: bfi x9, x8, #32, #32
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlv h0, v0.16b
+; GISEL-NEXT: // kill: def $q0 killed $h0
+; GISEL-NEXT: // kill: def $s0 killed $s0 killed $q0
+; GISEL-NEXT: fmov w0, s0
+; GISEL-NEXT: mov w8, w0
+; GISEL-NEXT: // kill: def $x10 killed $w10
+; GISEL-NEXT: bfi x8, x10, #32, #32
+; GISEL-NEXT: adds x8, x8, x9
+; GISEL-NEXT: mov w0, w8
+; GISEL-NEXT: ret
+;
; CHECK-LABEL: popcount256:
; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: ldr x11, [x0]
-; CHECK-NEXT: ldr x10, [x0, #8]
-; CHECK-NEXT: ldr x9, [x0, #16]
-; CHECK-NEXT: ldr x8, [x0, #24]
-; CHECK-NEXT: // implicit-def: $q1
-; CHECK-NEXT: mov v1.d[0], x11
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: ldp x8, x10, [x0]
+; CHECK-NEXT: mov v1.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
; CHECK-NEXT: mov v1.d[1], x10
-; CHECK-NEXT: // implicit-def: $q0
-; CHECK-NEXT: mov v0.d[0], x9
-; CHECK-NEXT: mov v0.d[1], x8
-; CHECK-NEXT: cnt v1.16b, v1.16b
-; CHECK-NEXT: uaddlv h1, v1.16b
-; CHECK-NEXT: // kill: def $q1 killed $h1
-; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
-; CHECK-NEXT: fmov w0, s1
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: mov w9, w0
-; CHECK-NEXT: mov w8, w10
-; CHECK-NEXT: bfi x9, x8, #32, #32
; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: cnt v1.16b, v1.16b
; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: mov w8, w0
-; CHECK-NEXT: // kill: def $x10 killed $w10
-; CHECK-NEXT: bfi x8, x10, #32, #32
-; CHECK-NEXT: adds x8, x8, x9
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: uaddlv h1, v1.16b
+; CHECK-NEXT: mov w8, v0.s[0]
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add x0, x8, w9, uxtw
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
Entry:
%1 = load i256, ptr %0, align 16
@@ -67,25 +140,338 @@ Entry:
declare i256 @llvm.ctpop.i256(i256)
define <1 x i128> @popcount1x128(<1 x i128> %0) {
+; CHECKO0-LABEL: popcount1x128:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: // implicit-def: $q0
+; CHECKO0-NEXT: mov v0.d[0], x0
+; CHECKO0-NEXT: mov v0.d[1], x1
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlv h0, v0.16b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: mov x1, xzr
+; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
+; CHECKO0-NEXT: fmov w0, s0
+; CHECKO0-NEXT: mov w8, wzr
+; CHECKO0-NEXT: // kill: def $x0 killed $w0
+; CHECKO0-NEXT: // kill: def $x8 killed $w8
+; CHECKO0-NEXT: bfi x0, x8, #32, #32
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount1x128:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: // implicit-def: $q0
+; GISEL-NEXT: mov v0.d[0], x0
+; GISEL-NEXT: mov v0.d[1], x1
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlv h0, v0.16b
+; GISEL-NEXT: // kill: def $q0 killed $h0
+; GISEL-NEXT: mov x1, xzr
+; GISEL-NEXT: // kill: def $s0 killed $s0 killed $q0
+; GISEL-NEXT: fmov w0, s0
+; GISEL-NEXT: mov w8, wzr
+; GISEL-NEXT: // kill: def $x0 killed $w0
+; GISEL-NEXT: // kill: def $x8 killed $w8
+; GISEL-NEXT: bfi x0, x8, #32, #32
+; GISEL-NEXT: ret
+;
; CHECK-LABEL: popcount1x128:
; CHECK: // %bb.0: // %Entry
-; CHECK-NEXT: // implicit-def: $q0
; CHECK-NEXT: mov v0.d[0], x0
; CHECK-NEXT: mov v0.d[1], x1
+; CHECK-NEXT: mov x1, xzr
; CHECK-NEXT: cnt v0.16b, v0.16b
; CHECK-NEXT: uaddlv h0, v0.16b
-; CHECK-NEXT: // kill: def $q0 killed $h0
-; CHECK-NEXT: mov x1, xzr
-; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: // kill: def $x0 killed $w0
-; CHECK-NEXT: // kill: def $x8 killed $w8
-; CHECK-NEXT: bfi x0, x8, #32, #32
+; CHECK-NEXT: mov w0, v0.s[0]
; CHECK-NEXT: ret
Entry:
- %1 = tail call <1 x i128> @llvm.ctpop.v1.i128(<1 x i128> %0)
+ %1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)
ret <1 x i128> %1
}
-declare <1 x i128> @llvm.ctpop.v1.i128(<1 x i128>)
+declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)
+
+define <2 x i64> @popcount2x64(<2 x i64> %0) {
+; CHECKO0-LABEL: popcount2x64:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
+; CHECKO0-NEXT: uaddlp v0.2d, v0.4s
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount2x64:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlp v0.8h, v0.16b
+; GISEL-NEXT: uaddlp v0.4s, v0.8h
+; GISEL-NEXT: uaddlp v0.2d, v0.4s
+; GISEL-NEXT: ret
+;
+; NEON-GISEL-LABEL: popcount2x64:
+; NEON-GISEL: // %bb.0: // %Entry
+; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
+; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
+; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
+; NEON-GISEL-NEXT: uaddlp v0.2d, v0.4s
+; NEON-GISEL-NEXT: ret
+;
+; DOT-GISEL-LABEL: popcount2x64:
+; DOT-GISEL: // %bb.0: // %Entry
+; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
+; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
+; DOT-GISEL-NEXT: movi v2.16b, #1
+; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
+; DOT-GISEL-NEXT: uaddlp v0.2d, v1.4s
+; DOT-GISEL-NEXT: ret
+;
+; SVE-GISEL-LABEL: popcount2x64:
+; SVE-GISEL: // %bb.0: // %Entry
+; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
+; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
+; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
+; SVE-GISEL-NEXT: uaddlp v0.2d, v0.4s
+; SVE-GISEL-NEXT: ret
+; NEON-LABEL: popcount2x64:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+; DOT-LABEL: popcount2x64:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.16b, #1
+; DOT-NEXT: cnt v0.16b, v0.16b
+; DOT-NEXT: movi v2.2d, #0000000000000000
+; DOT-NEXT: udot v2.4s, v1.16b, v0.16b
+; DOT-NEXT: uaddlp v0.2d, v2.4s
+; DOT-NEXT: ret
+; SVE-LABEL: popcount2x64:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlp v0.8h, v0.16b
+; SVE-NEXT: uaddlp v0.4s, v0.8h
+; SVE-NEXT: uaddlp v0.2d, v0.4s
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
+ ret <2 x i64> %1
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+
+define <1 x i64> @popcount1x64(<1 x i64> %0) {
+; CHECKO0-LABEL: popcount1x64:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: fmov x0, d0
+; CHECKO0-NEXT: fmov d0, x0
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlv h0, v0.8b
+; CHECKO0-NEXT: // kill: def $q0 killed $h0
+; CHECKO0-NEXT: mov w8, v0.s[0]
+; CHECKO0-NEXT: // kill: def $x8 killed $w8
+; CHECKO0-NEXT: fmov d0, x8
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount1x64:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: fmov x0, d0
+; GISEL-NEXT: fmov d0, x0
+; GISEL-NEXT: cnt v0.8b, v0.8b
+; GISEL-NEXT: uaddlv h0, v0.8b
+; GISEL-NEXT: // kill: def $q0 killed $h0
+; GISEL-NEXT: mov w8, v0.s[0]
+; GISEL-NEXT: // kill: def $x8 killed $w8
+; GISEL-NEXT: fmov d0, x8
+; GISEL-NEXT: ret
+;
+; CHECK-LABEL: popcount1x64:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlv h0, v0.8b
+; CHECK-NEXT: mov w8, v0.s[0]
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+Entry:
+ %1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0)
+ ret <1 x i64> %1
+}
+
+declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
+
+define <4 x i32> @popcount4x32(<4 x i32> %0) {
+; CHECKO0-LABEL: popcount4x32:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount4x32:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlp v0.8h, v0.16b
+; GISEL-NEXT: uaddlp v0.4s, v0.8h
+; GISEL-NEXT: ret
+;
+; NEON-GISEL-LABEL: popcount4x32:
+; NEON-GISEL: // %bb.0: // %Entry
+; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
+; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
+; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
+; NEON-GISEL-NEXT: ret
+;
+; DOT-GISEL-LABEL: popcount4x32:
+; DOT-GISEL: // %bb.0: // %Entry
+; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
+; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
+; DOT-GISEL-NEXT: movi v2.16b, #1
+; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
+; DOT-GISEL-NEXT: mov v0.16b, v1.16b
+; DOT-GISEL-NEXT: ret
+;
+; SVE-GISEL-LABEL: popcount4x32:
+; SVE-GISEL: // %bb.0: // %Entry
+; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
+; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
+; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
+; SVE-GISEL-NEXT: ret
+; NEON-LABEL: popcount4x32:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: ret
+; DOT-LABEL: popcount4x32:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.16b, #1
+; DOT-NEXT: cnt v2.16b, v0.16b
+; DOT-NEXT: movi v0.2d, #0000000000000000
+; DOT-NEXT: udot v0.4s, v1.16b, v2.16b
+; DOT-NEXT: ret
+; SVE-LABEL: popcount4x32:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: uaddlp v0.8h, v0.16b
+; SVE-NEXT: uaddlp v0.4s, v0.8h
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
+ ret <4 x i32> %1
+}
+
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
+
+define <2 x i32> @popcount2x32(<2 x i32> %0) {
+; CHECKO0-LABEL: popcount2x32:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
+; CHECKO0-NEXT: uaddlp v0.2s, v0.4h
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount2x32:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: cnt v0.8b, v0.8b
+; GISEL-NEXT: uaddlp v0.4h, v0.8b
+; GISEL-NEXT: uaddlp v0.2s, v0.4h
+; GISEL-NEXT: ret
+;
+; NEON-GISEL-LABEL: popcount2x32:
+; NEON-GISEL: // %bb.0: // %Entry
+; NEON-GISEL-NEXT: cnt v0.8b, v0.8b
+; NEON-GISEL-NEXT: uaddlp v0.4h, v0.8b
+; NEON-GISEL-NEXT: uaddlp v0.2s, v0.4h
+; NEON-GISEL-NEXT: ret
+;
+; DOT-GISEL-LABEL: popcount2x32:
+; DOT-GISEL: // %bb.0: // %Entry
+; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
+; DOT-GISEL-NEXT: cnt v0.8b, v0.8b
+; DOT-GISEL-NEXT: movi v2.8b, #1
+; DOT-GISEL-NEXT: udot v1.2s, v2.8b, v0.8b
+; DOT-GISEL-NEXT: fmov d0, d1
+; DOT-GISEL-NEXT: ret
+;
+; SVE-GISEL-LABEL: popcount2x32:
+; SVE-GISEL: // %bb.0: // %Entry
+; SVE-GISEL-NEXT: cnt v0.8b, v0.8b
+; SVE-GISEL-NEXT: uaddlp v0.4h, v0.8b
+; SVE-GISEL-NEXT: uaddlp v0.2s, v0.4h
+; SVE-GISEL-NEXT: ret
+; NEON-LABEL: popcount2x32:
+; NEON: // %bb.0: // %Entry
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: ret
+; DOT-LABEL: popcount2x32:
+; DOT: // %bb.0: // %Entry
+; DOT-NEXT: movi v1.2d, #0000000000000000
+; DOT-NEXT: cnt v0.8b, v0.8b
+; DOT-NEXT: movi v2.8b, #1
+; DOT-NEXT: udot v1.2s, v2.8b, v0.8b
+; DOT-NEXT: fmov d0, d1
+; DOT-NEXT: ret
+; SVE-LABEL: popcount2x32:
+; SVE: // %bb.0: // %Entry
+; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: uaddlp v0.4h, v0.8b
+; SVE-NEXT: uaddlp v0.2s, v0.4h
+; SVE-NEXT: ret
+Entry:
+ %1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)
+ ret <2 x i32> %1
+}
+
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+
+define <8 x i16> @popcount8x16(<8 x i16> %0) {
+; CHECKO0-LABEL: popcount8x16:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.16b, v0.16b
+; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount8x16:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: cnt v0.16b, v0.16b
+; GISEL-NEXT: uaddlp v0.8h, v0.16b
+; GISEL-NEXT: ret
+;
+; CHECK-LABEL: popcount8x16:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.16b, v0.16b
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+Entry:
+ %1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0)
+ ret <8 x i16> %1
+}
+
+declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
+
+define <4 x i16> @popcount4x16(<4 x i16> %0) {
+; CHECKO0-LABEL: popcount4x16:
+; CHECKO0: // %bb.0: // %Entry
+; CHECKO0-NEXT: cnt v0.8b, v0.8b
+; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
+; CHECKO0-NEXT: ret
+;
+; GISEL-LABEL: popcount4x16:
+; GISEL: // %bb.0: // %Entry
+; GISEL-NEXT: cnt v0.8b, v0.8b
+; GISEL-NEXT: uaddlp v0.4h, v0.8b
+; GISEL-NEXT: ret
+;
+; CHECK-LABEL: popcount4x16:
+; CHECK: // %bb.0: // %Entry
+; CHECK-NEXT: cnt v0.8b, v0.8b
+; CHECK-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-NEXT: ret
+Entry:
+ %1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0)
+ ret <4 x i16> %1
+}
+
+declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
>From d824063ff9d30e082d02c2c485736a42e594dc19 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 25 Jun 2024 14:45:40 +0200
Subject: [PATCH 3/4] remove unnecessary include and rename SUM to Sum
---
.../lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 8f127c418a36b..1ef9e78f6a527 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -14,7 +14,6 @@
#include "AArch64LegalizerInfo.h"
#include "AArch64RegisterBankInfo.h"
#include "AArch64Subtarget.h"
-#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
@@ -1911,21 +1910,21 @@ bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
LLT Dt = Ty == LLT::fixed_vector(2, 64) ? LLT::fixed_vector(4, 32) : Ty;
auto Zeros = MIRBuilder.buildConstant(Dt, 0);
auto Ones = MIRBuilder.buildConstant(VTy, 1);
- MachineInstrBuilder SUM;
+ MachineInstrBuilder Sum;
if (Ty == LLT::fixed_vector(2, 64)) {
auto UDOT =
MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
- SUM = MIRBuilder.buildInstr(AArch64::G_UADDLP, {Ty}, {UDOT});
+ Sum = MIRBuilder.buildInstr(AArch64::G_UADDLP, {Ty}, {UDOT});
} else if (Ty == LLT::fixed_vector(4, 32)) {
- SUM = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
+ Sum = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
} else if (Ty == LLT::fixed_vector(2, 32)) {
- SUM = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
+ Sum = MIRBuilder.buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones, CTPOP});
} else {
llvm_unreachable("unexpected vector shape");
}
- SUM->getOperand(0).setReg(Dst);
+ Sum->getOperand(0).setReg(Dst);
MI.eraseFromParent();
return true;
}
>From a8af97c5baf478986aaa1b1ee239fa8b18472b17 Mon Sep 17 00:00:00 2001
From: Tim Gymnich <tgymnich at icloud.com>
Date: Tue, 25 Jun 2024 14:46:17 +0200
Subject: [PATCH 4/4] remove old test check lines
---
llvm/test/CodeGen/AArch64/popcount.ll | 61 ---------------------------
1 file changed, 61 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index a5b32e8948f92..4abab47b6a365 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -231,28 +231,6 @@ define <2 x i64> @popcount2x64(<2 x i64> %0) {
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
; SVE-GISEL-NEXT: uaddlp v0.2d, v0.4s
; SVE-GISEL-NEXT: ret
-; NEON-LABEL: popcount2x64:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: cnt v0.16b, v0.16b
-; NEON-NEXT: uaddlp v0.8h, v0.16b
-; NEON-NEXT: uaddlp v0.4s, v0.8h
-; NEON-NEXT: uaddlp v0.2d, v0.4s
-; NEON-NEXT: ret
-; DOT-LABEL: popcount2x64:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: movi v1.16b, #1
-; DOT-NEXT: cnt v0.16b, v0.16b
-; DOT-NEXT: movi v2.2d, #0000000000000000
-; DOT-NEXT: udot v2.4s, v1.16b, v0.16b
-; DOT-NEXT: uaddlp v0.2d, v2.4s
-; DOT-NEXT: ret
-; SVE-LABEL: popcount2x64:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: cnt v0.16b, v0.16b
-; SVE-NEXT: uaddlp v0.8h, v0.16b
-; SVE-NEXT: uaddlp v0.4s, v0.8h
-; SVE-NEXT: uaddlp v0.2d, v0.4s
-; SVE-NEXT: ret
Entry:
%1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
ret <2 x i64> %1
@@ -336,25 +314,6 @@ define <4 x i32> @popcount4x32(<4 x i32> %0) {
; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
; SVE-GISEL-NEXT: ret
-; NEON-LABEL: popcount4x32:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: cnt v0.16b, v0.16b
-; NEON-NEXT: uaddlp v0.8h, v0.16b
-; NEON-NEXT: uaddlp v0.4s, v0.8h
-; NEON-NEXT: ret
-; DOT-LABEL: popcount4x32:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: movi v1.16b, #1
-; DOT-NEXT: cnt v2.16b, v0.16b
-; DOT-NEXT: movi v0.2d, #0000000000000000
-; DOT-NEXT: udot v0.4s, v1.16b, v2.16b
-; DOT-NEXT: ret
-; SVE-LABEL: popcount4x32:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: cnt v0.16b, v0.16b
-; SVE-NEXT: uaddlp v0.8h, v0.16b
-; SVE-NEXT: uaddlp v0.4s, v0.8h
-; SVE-NEXT: ret
Entry:
%1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
ret <4 x i32> %1
@@ -399,26 +358,6 @@ define <2 x i32> @popcount2x32(<2 x i32> %0) {
; SVE-GISEL-NEXT: uaddlp v0.4h, v0.8b
; SVE-GISEL-NEXT: uaddlp v0.2s, v0.4h
; SVE-GISEL-NEXT: ret
-; NEON-LABEL: popcount2x32:
-; NEON: // %bb.0: // %Entry
-; NEON-NEXT: cnt v0.8b, v0.8b
-; NEON-NEXT: uaddlp v0.4h, v0.8b
-; NEON-NEXT: uaddlp v0.2s, v0.4h
-; NEON-NEXT: ret
-; DOT-LABEL: popcount2x32:
-; DOT: // %bb.0: // %Entry
-; DOT-NEXT: movi v1.2d, #0000000000000000
-; DOT-NEXT: cnt v0.8b, v0.8b
-; DOT-NEXT: movi v2.8b, #1
-; DOT-NEXT: udot v1.2s, v2.8b, v0.8b
-; DOT-NEXT: fmov d0, d1
-; DOT-NEXT: ret
-; SVE-LABEL: popcount2x32:
-; SVE: // %bb.0: // %Entry
-; SVE-NEXT: cnt v0.8b, v0.8b
-; SVE-NEXT: uaddlp v0.4h, v0.8b
-; SVE-NEXT: uaddlp v0.2s, v0.4h
-; SVE-NEXT: ret
Entry:
%1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)
ret <2 x i32> %1
More information about the llvm-commits
mailing list