[llvm] [CodeGen] Optimize ctpop expansion using computeKnownBits (PR #221671)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 02:55:55 PDT 2026
https://github.com/milkHongYe updated https://github.com/llvm/llvm-project/pull/221671
>From 9d482cb71bf94c0c5ab3ccade76ad856044d8e44 Mon Sep 17 00:00:00 2001
From: milkHongYe <1483685464 at qq.com>
Date: Mon, 7 Sep 2026 15:10:37 +0800
Subject: [PATCH 1/3] [CodeGen] Optimize ctpop expansion using computeKnownBits
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 13 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 18 +-
...treaming-mode-fixed-length-bit-counting.ll | 2280 ++++++++---------
.../AMDGPU/bit-op-reduce-width-known-bits.ll | 6 +-
llvm/test/CodeGen/PowerPC/popcnt-zext.ll | 23 +-
.../CodeGen/RISCV/ctz_zero_return_test.ll | 26 +-
llvm/test/CodeGen/RISCV/pr56457.ll | 31 +-
llvm/test/CodeGen/X86/ctpop-mask.ll | 12 +-
llvm/test/CodeGen/X86/masked_compressstore.ll | 6 +-
llvm/test/CodeGen/X86/masked_expandload.ll | 6 +-
llvm/test/CodeGen/X86/popcnt.ll | 17 +-
11 files changed, 1161 insertions(+), 1277 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b2605ef54a9ea..bfe1b4f80bf64 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12330,6 +12330,19 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
OutVal, SDNoFPClass);
}
+ if (VT.isInteger()) {
+ if (auto CR = Arg.getRange()) {
+ if (!CR->isFullSet() && !CR->isEmptySet() &&
+ !CR->isUpperWrapped()) {
+ APInt Hi = CR->getUnsignedMax();
+ unsigned Bits = std::max(Hi.getActiveBits(),
+ (unsigned)IntegerType::MIN_INT_BITS);
+ EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
+ OutVal = DAG.getNode(ISD::AssertZext, dl, OutVal.getValueType(),
+ OutVal, DAG.getValueType(SmallVT));
+ }
+ }
+ }
ArgValues.push_back(OutVal);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index cca7cdad0e2c8..9c83b698144b5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -10825,6 +10825,18 @@ SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Op = Node->getOperand(0);
unsigned Len = VT.getScalarSizeInBits();
+
+ // Compute effective bit width from known bits
+ KnownBits Known = DAG.computeKnownBits(Op);
+ unsigned EffectiveLen = Known.countMaxActiveBits();
+
+ // Round up to 8-bit boundary for byte-oriented SWAR algorithm
+ if (EffectiveLen > 0 && EffectiveLen < Len) {
+ EffectiveLen = (EffectiveLen + 7) & ~7u;
+ } else {
+ EffectiveLen = Len;
+ }
+
assert(VT.isInteger() && "CTPOP not implemented for this type.");
// TODO: Add support for irregular type lengths.
@@ -10863,13 +10875,13 @@ SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
DAG.getConstant(4, dl, ShVT))),
Mask0F);
- if (Len <= 8)
+ if (EffectiveLen <= 8)
return Op;
// Avoid the multiply if we only have 2 bytes to add.
// TODO: Only doing this for scalars because vectors weren't as obviously
// improved.
- if (Len == 16 && !VT.isVector()) {
+ if (EffectiveLen == 16 && !VT.isVector()) {
// v = (v + (v >> 8)) & 0x00FF;
return DAG.getNode(ISD::AND, dl, VT,
DAG.getNode(ISD::ADD, dl, VT, Op,
@@ -10887,7 +10899,7 @@ SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
V = DAG.getNode(ISD::MUL, dl, VT, Op, Mask01);
} else {
V = Op;
- for (unsigned Shift = 8; Shift < Len; Shift *= 2) {
+ for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl);
V = DAG.getNode(ISD::ADD, dl, VT, V,
DAG.getNode(ISD::SHL, dl, VT, V, ShiftC));
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
index baaf8c3d906c9..8a55b72b3ba3c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
@@ -720,59 +720,50 @@ define <4 x i8> @ctpop_v4i8(<4 x i8> %op) {
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
-; NONEON-NOSVE-NEXT: ldrb w11, [sp, #4]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: ldrb w10, [sp, #2]
-; NONEON-NOSVE-NEXT: ldrb w12, [sp]
-; NONEON-NOSVE-NEXT: lsr w13, w9, #1
-; NONEON-NOSVE-NEXT: lsr w14, w11, #1
-; NONEON-NOSVE-NEXT: lsr w15, w10, #1
-; NONEON-NOSVE-NEXT: lsr w16, w12, #1
-; NONEON-NOSVE-NEXT: and w13, w13, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w13
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; NONEON-NOSVE-NEXT: ldrb w10, [sp, #4]
+; NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
+; NONEON-NOSVE-NEXT: ldrb w11, [sp]
+; NONEON-NOSVE-NEXT: lsr w12, w8, #1
+; NONEON-NOSVE-NEXT: lsr w13, w10, #1
+; NONEON-NOSVE-NEXT: lsr w14, w9, #1
+; NONEON-NOSVE-NEXT: lsr w15, w11, #1
+; NONEON-NOSVE-NEXT: and w12, w12, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w12
+; NONEON-NOSVE-NEXT: and w12, w13, #0x55555555
; NONEON-NOSVE-NEXT: and w13, w14, #0x55555555
+; NONEON-NOSVE-NEXT: sub w10, w10, w12
+; NONEON-NOSVE-NEXT: lsr w12, w8, #2
; NONEON-NOSVE-NEXT: and w14, w15, #0x55555555
-; NONEON-NOSVE-NEXT: sub w11, w11, w13
-; NONEON-NOSVE-NEXT: lsr w13, w9, #2
-; NONEON-NOSVE-NEXT: and w15, w16, #0x55555555
-; NONEON-NOSVE-NEXT: sub w10, w10, w14
-; NONEON-NOSVE-NEXT: sub w12, w12, w15
+; NONEON-NOSVE-NEXT: sub w9, w9, w13
+; NONEON-NOSVE-NEXT: sub w11, w11, w14
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w12, w12, #0x33333333
+; NONEON-NOSVE-NEXT: lsr w13, w10, #2
+; NONEON-NOSVE-NEXT: lsr w14, w9, #2
+; NONEON-NOSVE-NEXT: add w8, w8, w12
+; NONEON-NOSVE-NEXT: lsr w12, w11, #2
+; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
; NONEON-NOSVE-NEXT: and w13, w13, #0x33333333
-; NONEON-NOSVE-NEXT: lsr w14, w11, #2
-; NONEON-NOSVE-NEXT: lsr w15, w10, #2
-; NONEON-NOSVE-NEXT: add w9, w9, w13
-; NONEON-NOSVE-NEXT: lsr w13, w12, #2
-; NONEON-NOSVE-NEXT: and w11, w11, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
; NONEON-NOSVE-NEXT: and w14, w14, #0x33333333
-; NONEON-NOSVE-NEXT: and w15, w15, #0x33333333
+; NONEON-NOSVE-NEXT: and w11, w11, #0x33333333
; NONEON-NOSVE-NEXT: and w12, w12, #0x33333333
-; NONEON-NOSVE-NEXT: and w13, w13, #0x33333333
-; NONEON-NOSVE-NEXT: add w11, w11, w14
-; NONEON-NOSVE-NEXT: add w10, w10, w15
-; NONEON-NOSVE-NEXT: add w12, w12, w13
+; NONEON-NOSVE-NEXT: add w10, w10, w13
+; NONEON-NOSVE-NEXT: add w9, w9, w14
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: add w11, w11, w12
+; NONEON-NOSVE-NEXT: add w10, w10, w10, lsr #4
; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
; NONEON-NOSVE-NEXT: add w11, w11, w11, lsr #4
-; NONEON-NOSVE-NEXT: add w10, w10, w10, lsr #4
-; NONEON-NOSVE-NEXT: add w12, w12, w12, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: and w11, w11, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
; NONEON-NOSVE-NEXT: and w10, w10, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: and w12, w12, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: mul w11, w11, w8
-; NONEON-NOSVE-NEXT: mul w10, w10, w8
-; NONEON-NOSVE-NEXT: mul w8, w12, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: lsr w11, w11, #24
-; NONEON-NOSVE-NEXT: lsr w10, w10, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #14]
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
-; NONEON-NOSVE-NEXT: strh w11, [sp, #12]
-; NONEON-NOSVE-NEXT: strh w10, [sp, #10]
-; NONEON-NOSVE-NEXT: strh w8, [sp, #8]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: and w8, w9, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: and w9, w11, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: strh w10, [sp, #12]
+; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: strh w9, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
@@ -791,110 +782,93 @@ define <8 x i8> @ctpop_v8i8(<8 x i8> %op) {
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #7]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #15]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #14]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #5]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #13]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #12]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #3]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #11]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #10]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #1]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #9]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #12]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
; NONEON-NOSVE-NEXT: strb w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -914,214 +888,181 @@ define <16 x i8> @ctpop_v16i8(<16 x i8> %op) {
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #15]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #31]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #14]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #30]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #13]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #29]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #12]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #28]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #11]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #27]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #10]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #26]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #9]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #25]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #8]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #24]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #7]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #23]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #22]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #5]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #21]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #20]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #3]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #19]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #18]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #1]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #17]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #31]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #30]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #29]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #28]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #27]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #26]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #25]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #24]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #23]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #22]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #21]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #20]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #19]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #18]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #17]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
; NONEON-NOSVE-NEXT: strb w8, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -1143,473 +1084,407 @@ define void @ctpop_v32i8(ptr %a) {
; NONEON-NOSVE-LABEL: ctpop_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #31]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #63]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #30]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #62]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #29]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #61]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #28]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #60]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #27]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #59]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #26]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #58]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #25]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: ret
+ %op = load <32 x i8>, ptr %a
+ %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define <2 x i16> @ctpop_v2i16(<2 x i16> %op) {
+; CHECK-LABEL: ctpop_v2i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: cnt z0.s, p0/m, z0.s
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: ctpop_v2i16:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp]
+; NONEON-NOSVE-NEXT: lsr w10, w8, #1
+; NONEON-NOSVE-NEXT: lsr w11, w9, #1
; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
+; NONEON-NOSVE-NEXT: and w11, w11, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w10
+; NONEON-NOSVE-NEXT: sub w9, w9, w11
+; NONEON-NOSVE-NEXT: lsr w10, w8, #2
+; NONEON-NOSVE-NEXT: lsr w11, w9, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w10
+; NONEON-NOSVE-NEXT: and w10, w11, #0x33333333
; NONEON-NOSVE-NEXT: add w9, w9, w10
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #57]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #24]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #56]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #23]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #55]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #22]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #54]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #21]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #53]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #20]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #52]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #19]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #51]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #18]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #50]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #17]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #49]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #16]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #48]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #15]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #47]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #14]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #46]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #13]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #45]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #12]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #44]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #11]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #43]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #10]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #42]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #9]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #41]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #8]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #40]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #7]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #39]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #38]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #5]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #37]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #36]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #3]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #35]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #34]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp, #1]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strb w9, [sp, #33]
-; NONEON-NOSVE-NEXT: ldrb w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
-; NONEON-NOSVE-NEXT: strb w8, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
-; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
-; NONEON-NOSVE-NEXT: ret
- %op = load <32 x i8>, ptr %a
- %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
- store <32 x i8> %res, ptr %a
- ret void
-}
-
-define <2 x i16> @ctpop_v2i16(<2 x i16> %op) {
-; CHECK-LABEL: ctpop_v2i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: cnt z0.s, p0/m, z0.s
-; CHECK-NEXT: ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v2i16:
-; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
-; NONEON-NOSVE-NEXT: ldrh w10, [sp]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: lsr w11, w9, #1
-; NONEON-NOSVE-NEXT: lsr w12, w10, #1
-; NONEON-NOSVE-NEXT: and w11, w11, #0x55555555
-; NONEON-NOSVE-NEXT: and w12, w12, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w11
-; NONEON-NOSVE-NEXT: sub w10, w10, w12
-; NONEON-NOSVE-NEXT: lsr w11, w9, #2
-; NONEON-NOSVE-NEXT: lsr w12, w10, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: and w11, w11, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w11
-; NONEON-NOSVE-NEXT: and w11, w12, #0x33333333
-; NONEON-NOSVE-NEXT: add w10, w10, w11
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: add w10, w10, w10, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: and w10, w10, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: mul w8, w10, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: and w9, w9, #0x1f
+; NONEON-NOSVE-NEXT: stp w9, w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
@@ -1622,187 +1497,185 @@ define <4 x i16> @ctpop_v4i16(<4 x i16> %op) {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: cnt z0.h, p0/m, z0.h
-; CHECK-NEXT: ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v4i16:
-; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #14]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #12]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #10]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
-; NONEON-NOSVE-NEXT: strh w8, [sp, #8]
-; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
-; NONEON-NOSVE-NEXT: ret
- %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
- ret <4 x i16> %res
-}
-
-define <8 x i16> @ctpop_v8i16(<8 x i16> %op) {
-; CHECK-LABEL: ctpop_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: cnt z0.h, p0/m, z0.h
-; CHECK-NEXT: ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v8i16:
-; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #14]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #30]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #12]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #28]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #10]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #26]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #24]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #22]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #20]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #18]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: ctpop_v4i16:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #12]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
+; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: ret
+ %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @ctpop_v8i16(<8 x i16> %op) {
+; CHECK-LABEL: ctpop_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: cnt z0.h, p0/m, z0.h
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: ctpop_v8i16:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #30]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #28]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #26]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #24]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #22]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #20]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #18]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
; NONEON-NOSVE-NEXT: strh w8, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -1824,216 +1697,215 @@ define void @ctpop_v16i16(ptr %a) {
; NONEON-NOSVE-LABEL: ctpop_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT: mov w8, #16843009 // =0x1010101
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #30]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #62]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #28]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #60]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #26]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #58]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #24]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #56]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #22]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #54]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #20]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #52]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #18]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #50]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #16]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #48]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #14]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #46]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #12]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #44]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #10]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #42]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #40]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #38]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #36]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w9, w9, w8
-; NONEON-NOSVE-NEXT: lsr w9, w9, #24
-; NONEON-NOSVE-NEXT: strh w9, [sp, #34]
-; NONEON-NOSVE-NEXT: ldrh w9, [sp]
-; NONEON-NOSVE-NEXT: lsr w10, w9, #1
-; NONEON-NOSVE-NEXT: and w10, w10, #0x55555555
-; NONEON-NOSVE-NEXT: sub w9, w9, w10
-; NONEON-NOSVE-NEXT: lsr w10, w9, #2
-; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
-; NONEON-NOSVE-NEXT: and w10, w10, #0x33333333
-; NONEON-NOSVE-NEXT: add w9, w9, w10
-; NONEON-NOSVE-NEXT: add w9, w9, w9, lsr #4
-; NONEON-NOSVE-NEXT: and w9, w9, #0xf0f0f0f
-; NONEON-NOSVE-NEXT: mul w8, w9, w8
-; NONEON-NOSVE-NEXT: lsr w8, w8, #24
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; NONEON-NOSVE-NEXT: lsr w9, w8, #1
+; NONEON-NOSVE-NEXT: and w9, w9, #0x55555555
+; NONEON-NOSVE-NEXT: sub w8, w8, w9
+; NONEON-NOSVE-NEXT: lsr w9, w8, #2
+; NONEON-NOSVE-NEXT: and w8, w8, #0x33333333
+; NONEON-NOSVE-NEXT: and w9, w9, #0x33333333
+; NONEON-NOSVE-NEXT: add w8, w8, w9
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #4
+; NONEON-NOSVE-NEXT: and w8, w8, #0xf0f0f0f
+; NONEON-NOSVE-NEXT: add w8, w8, w8, lsr #8
+; NONEON-NOSVE-NEXT: and w8, w8, #0x1f
; NONEON-NOSVE-NEXT: strh w8, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
diff --git a/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll b/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
index c3f4d1d633376..f7ede797d035d 100644
--- a/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
@@ -11,7 +11,7 @@ define i64 @v_xor_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %
; CHECK-LABEL: v_xor_i64_known_hi_i32_from_arg_range:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_xor_b32_e32 v1, v1, v3
+; CHECK-NEXT: v_mov_b32_e32 v1, v3
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: s_setpc_b64 s[30:31]
%xor = xor i64 %arg0, %arg1
@@ -24,7 +24,7 @@ define i64 @v_or_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %a
; CHECK-LABEL: v_or_i64_known_hi_i32_from_arg_range:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v3
+; CHECK-NEXT: v_mov_b32_e32 v1, v3
; CHECK-NEXT: v_or_b32_e32 v0, v0, v2
; CHECK-NEXT: s_setpc_b64 s[30:31]
%or = or i64 %arg0, %arg1
@@ -50,7 +50,7 @@ define i64 @s_xor_i64_known_i32_from_arg_range(i64 range(i64 0, 65) inreg %arg)
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_not_b64 s[4:5], s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, s4
-; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mov_b32_e32 v1, -1
; CHECK-NEXT: s_setpc_b64 s[30:31]
%xor = xor i64 %arg, -1
ret i64 %xor
diff --git a/llvm/test/CodeGen/PowerPC/popcnt-zext.ll b/llvm/test/CodeGen/PowerPC/popcnt-zext.ll
index d79edb5fc5d37..5faa1604fc3ed 100644
--- a/llvm/test/CodeGen/PowerPC/popcnt-zext.ll
+++ b/llvm/test/CodeGen/PowerPC/popcnt-zext.ll
@@ -23,9 +23,7 @@ define i16 @zpop_i8_i16(i8 %x) {
; SLOW-NEXT: add 3, 4, 3
; SLOW-NEXT: srwi 4, 3, 4
; SLOW-NEXT: add 3, 3, 4
-; SLOW-NEXT: rlwinm 4, 3, 24, 28, 31
-; SLOW-NEXT: clrlwi 3, 3, 28
-; SLOW-NEXT: add 3, 3, 4
+; SLOW-NEXT: andi. 3, 3, 3855
; SLOW-NEXT: blr
%z = zext i8 %x to i16
%pop = tail call i16 @llvm.ctpop.i16(i16 %z)
@@ -81,12 +79,8 @@ define i32 @zpop_i8_i32(i8 %x) {
; SLOW-NEXT: lis 5, 3855
; SLOW-NEXT: srwi 4, 3, 4
; SLOW-NEXT: add 3, 3, 4
-; SLOW-NEXT: lis 4, 257
-; SLOW-NEXT: ori 5, 5, 3855
-; SLOW-NEXT: and 3, 3, 5
-; SLOW-NEXT: ori 4, 4, 257
-; SLOW-NEXT: mullw 3, 3, 4
-; SLOW-NEXT: srwi 3, 3, 24
+; SLOW-NEXT: ori 4, 5, 3855
+; SLOW-NEXT: and 3, 3, 4
; SLOW-NEXT: blr
%z = zext i8 %x to i32
%pop = tail call i32 @llvm.ctpop.i32(i32 %z)
@@ -139,15 +133,16 @@ define i32 @zpop_i16_i32(i16 %x) {
; SLOW-NEXT: andi. 5, 5, 13107
; SLOW-NEXT: or 4, 5, 4
; SLOW-NEXT: add 3, 3, 4
-; SLOW-NEXT: lis 5, 3855
; SLOW-NEXT: srwi 4, 3, 4
; SLOW-NEXT: add 3, 3, 4
-; SLOW-NEXT: lis 4, 257
+; SLOW-NEXT: lis 5, 3855
+; SLOW-NEXT: rlwinm 4, 3, 24, 28, 31
; SLOW-NEXT: ori 5, 5, 3855
+; SLOW-NEXT: rlwimi 4, 3, 24, 20, 23
+; SLOW-NEXT: rlwimi 4, 3, 24, 12, 15
; SLOW-NEXT: and 3, 3, 5
-; SLOW-NEXT: ori 4, 4, 257
-; SLOW-NEXT: mullw 3, 3, 4
-; SLOW-NEXT: srwi 3, 3, 24
+; SLOW-NEXT: add 3, 3, 4
+; SLOW-NEXT: clrlwi 3, 3, 27
; SLOW-NEXT: blr
%z = zext i16 %x to i32
%pop = tail call i32 @llvm.ctpop.i32(i32 %z)
diff --git a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
index 290390b7edaf4..6e1b4eb7e9ede 100644
--- a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
+++ b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
@@ -1089,11 +1089,6 @@ define i7 @test_cttz_select_i7(i7 %0) {
; RV32I-NEXT: add a0, a0, a2
; RV32I-NEXT: addi a2, a3, -241
; RV32I-NEXT: and a0, a0, a2
-; RV32I-NEXT: slli a2, a0, 8
-; RV32I-NEXT: add a0, a0, a2
-; RV32I-NEXT: slli a2, a0, 16
-; RV32I-NEXT: add a0, a0, a2
-; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: j .LBB11_3
; RV32I-NEXT: .LBB11_2:
; RV32I-NEXT: li a0, 7
@@ -1130,13 +1125,6 @@ define i7 @test_cttz_select_i7(i7 %0) {
; RV64I-NEXT: add a0, a0, a3
; RV64I-NEXT: add a2, a2, a4
; RV64I-NEXT: and a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 8
-; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 16
-; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 32
-; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: j .LBB11_3
; RV64I-NEXT: .LBB11_2:
; RV64I-NEXT: li a0, 7
@@ -1292,11 +1280,9 @@ define i10 @test_ctlz_select_i10(i10 %0) {
; RV32I-NEXT: add a0, a0, a2
; RV32I-NEXT: addi a2, a3, -241
; RV32I-NEXT: and a0, a0, a2
-; RV32I-NEXT: slli a2, a0, 8
-; RV32I-NEXT: add a0, a0, a2
-; RV32I-NEXT: slli a2, a0, 16
+; RV32I-NEXT: srli a2, a0, 8
; RV32I-NEXT: add a0, a0, a2
-; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: andi a0, a0, 31
; RV32I-NEXT: j .LBB12_3
; RV32I-NEXT: .LBB12_2:
; RV32I-NEXT: li a0, 10
@@ -1342,13 +1328,9 @@ define i10 @test_ctlz_select_i10(i10 %0) {
; RV64I-NEXT: add a0, a0, a3
; RV64I-NEXT: add a2, a2, a4
; RV64I-NEXT: and a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 8
-; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 16
-; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: slli a2, a0, 32
+; RV64I-NEXT: srli a2, a0, 8
; RV64I-NEXT: add a0, a0, a2
-; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: andi a0, a0, 31
; RV64I-NEXT: j .LBB12_3
; RV64I-NEXT: .LBB12_2:
; RV64I-NEXT: li a0, 10
diff --git a/llvm/test/CodeGen/RISCV/pr56457.ll b/llvm/test/CodeGen/RISCV/pr56457.ll
index 3e67794e98f06..5d0d401b131fa 100644
--- a/llvm/test/CodeGen/RISCV/pr56457.ll
+++ b/llvm/test/CodeGen/RISCV/pr56457.ll
@@ -13,40 +13,37 @@ define i15 @foo(i15 %x) nounwind {
; CHECK-NEXT: srli a1, a1, 51
; CHECK-NEXT: or a0, a0, a1
; CHECK-NEXT: slli a1, a0, 49
-; CHECK-NEXT: lui a2, 1
; CHECK-NEXT: srli a1, a1, 53
+; CHECK-NEXT: or a0, a0, a1
+; CHECK-NEXT: slli a1, a0, 49
+; CHECK-NEXT: lui a2, 1
+; CHECK-NEXT: srli a1, a1, 57
; CHECK-NEXT: lui a3, 209715
; CHECK-NEXT: or a0, a0, a1
; CHECK-NEXT: addi a1, a2, 1365
-; CHECK-NEXT: slli a2, a0, 49
-; CHECK-NEXT: addi a3, a3, 819
-; CHECK-NEXT: srli a2, a2, 57
-; CHECK-NEXT: or a0, a0, a2
-; CHECK-NEXT: slli a2, a3, 32
-; CHECK-NEXT: add a2, a3, a2
; CHECK-NEXT: not a0, a0
+; CHECK-NEXT: addi a2, a3, 819
; CHECK-NEXT: srli a3, a0, 1
; CHECK-NEXT: slli a0, a0, 49
; CHECK-NEXT: and a1, a3, a1
; CHECK-NEXT: srli a0, a0, 49
-; CHECK-NEXT: lui a3, 61681
+; CHECK-NEXT: slli a3, a2, 32
; CHECK-NEXT: sub a0, a0, a1
+; CHECK-NEXT: add a2, a2, a3
; CHECK-NEXT: and a1, a0, a2
; CHECK-NEXT: srli a0, a0, 2
; CHECK-NEXT: and a0, a0, a2
-; CHECK-NEXT: lui a2, 4112
-; CHECK-NEXT: addi a3, a3, -241
+; CHECK-NEXT: lui a2, 61681
; CHECK-NEXT: add a0, a1, a0
-; CHECK-NEXT: addi a1, a2, 257
+; CHECK-NEXT: addi a1, a2, -241
; CHECK-NEXT: srli a2, a0, 4
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: slli a2, a3, 32
-; CHECK-NEXT: add a2, a3, a2
; CHECK-NEXT: slli a3, a1, 32
-; CHECK-NEXT: and a0, a0, a2
+; CHECK-NEXT: add a0, a0, a2
; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: mul a0, a0, a1
-; CHECK-NEXT: srli a0, a0, 56
+; CHECK-NEXT: and a0, a0, a1
+; CHECK-NEXT: srli a1, a0, 8
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: andi a0, a0, 31
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB0_2:
; CHECK-NEXT: li a0, 15
diff --git a/llvm/test/CodeGen/X86/ctpop-mask.ll b/llvm/test/CodeGen/X86/ctpop-mask.ll
index a43dba94d30c7..93bdeb6ed1944 100644
--- a/llvm/test/CodeGen/X86/ctpop-mask.ll
+++ b/llvm/test/CodeGen/X86/ctpop-mask.ll
@@ -609,8 +609,10 @@ define i32 @ctpop_mask16(i32 %x) nounwind readnone {
; X86-NO-POPCOUNT-NEXT: shrl $4, %ecx
; X86-NO-POPCOUNT-NEXT: addl %eax, %ecx
; X86-NO-POPCOUNT-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NO-POPCOUNT-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101
-; X86-NO-POPCOUNT-NEXT: shrl $24, %eax
+; X86-NO-POPCOUNT-NEXT: movl %ecx, %eax
+; X86-NO-POPCOUNT-NEXT: shrl $8, %eax
+; X86-NO-POPCOUNT-NEXT: addl %ecx, %eax
+; X86-NO-POPCOUNT-NEXT: movzbl %al, %eax
; X86-NO-POPCOUNT-NEXT: retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask16:
@@ -628,8 +630,10 @@ define i32 @ctpop_mask16(i32 %x) nounwind readnone {
; X64-NO-POPCOUNT-NEXT: shrl $4, %ecx
; X64-NO-POPCOUNT-NEXT: addl %eax, %ecx
; X64-NO-POPCOUNT-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
-; X64-NO-POPCOUNT-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101
-; X64-NO-POPCOUNT-NEXT: shrl $24, %eax
+; X64-NO-POPCOUNT-NEXT: movl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: shrl $8, %eax
+; X64-NO-POPCOUNT-NEXT: addl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: movzbl %al, %eax
; X64-NO-POPCOUNT-NEXT: retq
%mask = and i32 %x, 65535
%count = tail call i32 @llvm.ctpop.i32(i32 %mask)
diff --git a/llvm/test/CodeGen/X86/masked_compressstore.ll b/llvm/test/CodeGen/X86/masked_compressstore.ll
index 5296c9d0f0777..bb5fe9ec0c979 100644
--- a/llvm/test/CodeGen/X86/masked_compressstore.ll
+++ b/llvm/test/CodeGen/X86/masked_compressstore.ll
@@ -2415,8 +2415,10 @@ define void @compressstore_v32f32_v32i32(ptr %base, <32 x float> %V, <32 x i32>
; AVX512-NEXT: shrl $4, %ecx
; AVX512-NEXT: addl %eax, %ecx
; AVX512-NEXT: andl $252645135, %ecx ## imm = 0xF0F0F0F
-; AVX512-NEXT: imull $16843009, %ecx, %eax ## imm = 0x1010101
-; AVX512-NEXT: shrl $24, %eax
+; AVX512-NEXT: movl %ecx, %eax
+; AVX512-NEXT: shrl $8, %eax
+; AVX512-NEXT: addl %ecx, %eax
+; AVX512-NEXT: movzbl %al, %eax
; AVX512-NEXT: vcompressps %zmm1, (%rdi,%rax,4) {%k1}
; AVX512-NEXT: vcompressps %zmm0, (%rdi) {%k2}
; AVX512-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/masked_expandload.ll b/llvm/test/CodeGen/X86/masked_expandload.ll
index ce8a34db498df..a3ac72d61768d 100644
--- a/llvm/test/CodeGen/X86/masked_expandload.ll
+++ b/llvm/test/CodeGen/X86/masked_expandload.ll
@@ -2620,8 +2620,10 @@ define <32 x float> @expandload_v32f32_v32i32(ptr %base, <32 x float> %src0, <32
; AVX512-NEXT: shrl $4, %ecx
; AVX512-NEXT: addl %eax, %ecx
; AVX512-NEXT: andl $252645135, %ecx ## imm = 0xF0F0F0F
-; AVX512-NEXT: imull $16843009, %ecx, %eax ## imm = 0x1010101
-; AVX512-NEXT: shrl $24, %eax
+; AVX512-NEXT: movl %ecx, %eax
+; AVX512-NEXT: shrl $8, %eax
+; AVX512-NEXT: addl %ecx, %eax
+; AVX512-NEXT: movzbl %al, %eax
; AVX512-NEXT: vexpandps (%rdi,%rax,4), %zmm1 {%k2}
; AVX512-NEXT: vexpandps (%rdi), %zmm0 {%k1}
; AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/popcnt.ll b/llvm/test/CodeGen/X86/popcnt.ll
index 3004b8b72fcc5..500c82367fa45 100644
--- a/llvm/test/CodeGen/X86/popcnt.ll
+++ b/llvm/test/CodeGen/X86/popcnt.ll
@@ -1766,8 +1766,10 @@ define i32 @popcount_zext_i32(i16 zeroext %x) {
; X86-NEXT: shrl $4, %ecx
; X86-NEXT: addl %eax, %ecx
; X86-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101
-; X86-NEXT: shrl $24, %eax
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $8, %eax
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: movzbl %al, %eax
; X86-NEXT: retl
;
; X64-BASE-LABEL: popcount_zext_i32:
@@ -1785,8 +1787,10 @@ define i32 @popcount_zext_i32(i16 zeroext %x) {
; X64-BASE-NEXT: shrl $4, %eax
; X64-BASE-NEXT: addl %edi, %eax
; X64-BASE-NEXT: andl $252645135, %eax # imm = 0xF0F0F0F
-; X64-BASE-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101
-; X64-BASE-NEXT: shrl $24, %eax
+; X64-BASE-NEXT: movl %eax, %ecx
+; X64-BASE-NEXT: shrl $8, %ecx
+; X64-BASE-NEXT: addl %eax, %ecx
+; X64-BASE-NEXT: movzbl %cl, %eax
; X64-BASE-NEXT: retq
;
; X86-POPCNT-LABEL: popcount_zext_i32:
@@ -1812,8 +1816,9 @@ define i32 @popcount_zext_i32(i16 zeroext %x) {
; X64-NDD-NEXT: shrl $4, %eax, %ecx
; X64-NDD-NEXT: addl %ecx, %eax
; X64-NDD-NEXT: andl $252645135, %eax # imm = 0xF0F0F0F
-; X64-NDD-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101
-; X64-NDD-NEXT: shrl $24, %eax
+; X64-NDD-NEXT: shrl $8, %eax, %ecx
+; X64-NDD-NEXT: addl %ecx, %eax
+; X64-NDD-NEXT: movzbl %al, %eax
; X64-NDD-NEXT: retq
%z = zext i16 %x to i32
%cnt = tail call i32 @llvm.ctpop.i32(i32 %z)
>From 93e9f03c01f5912034de4703b823c4bf9aca6bd4 Mon Sep 17 00:00:00 2001
From: milkHongYe <1483685464 at qq.com>
Date: Tue, 8 Sep 2026 19:22:50 +0800
Subject: [PATCH 2/3] Shift active bits down in expandCTPOP when trailing zeros
are known
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 21 +++++---
llvm/test/CodeGen/X86/ctpop-mask.ll | 52 ++++++++++---------
2 files changed, 41 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 1f1dcd031017e..9156112b4a205 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -10799,17 +10799,24 @@ SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
SDValue Op = Node->getOperand(0);
unsigned Len = VT.getScalarSizeInBits();
- // Compute effective bit width from known bits
+ // Compute effective bit width from known bits, allowing us to shift the
+ // active bits down if necessary to fit into smaller specialized expansions.
KnownBits Known = DAG.computeKnownBits(Op);
- unsigned EffectiveLen = Known.countMaxActiveBits();
+ unsigned LZ = Known.countMinLeadingZeros();
+ unsigned TZ = Known.countMinTrailingZeros();
+ unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
- // Round up to 8-bit boundary for byte-oriented SWAR algorithm
- if (EffectiveLen > 0 && EffectiveLen < Len) {
- EffectiveLen = (EffectiveLen + 7) & ~7u;
- } else {
- EffectiveLen = Len;
+ // If the active bits are not at the low end, shift them down
+ if (ShiftedActiveBits < Len && TZ > 0) {
+ Op = DAG.getNode(ISD::SRL, dl, VT, Op,
+ DAG.getShiftAmountConstant(TZ, VT, dl));
}
+ // Round up to 8-bit boundary for byte-oriented SWAR algorithm
+ unsigned EffectiveLen = Len;
+ if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
+ EffectiveLen = std::min(alignTo(ShiftedActiveBits, 8), Len);
+
assert(VT.isInteger() && "CTPOP not implemented for this type.");
// TODO: Add support for irregular type lengths.
diff --git a/llvm/test/CodeGen/X86/ctpop-mask.ll b/llvm/test/CodeGen/X86/ctpop-mask.ll
index 93bdeb6ed1944..4eda6da04ed43 100644
--- a/llvm/test/CodeGen/X86/ctpop-mask.ll
+++ b/llvm/test/CodeGen/X86/ctpop-mask.ll
@@ -661,11 +661,12 @@ define i64 @ctpop_shifted_mask16(i64 %x) nounwind readnone {
; X86-NO-POPCOUNT-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-POPCOUNT-NEXT: movl %ecx, %eax
; X86-NO-POPCOUNT-NEXT: andl $524280, %eax # imm = 0x7FFF8
-; X86-NO-POPCOUNT-NEXT: shrl %ecx
-; X86-NO-POPCOUNT-NEXT: andl $87380, %ecx # imm = 0x15554
+; X86-NO-POPCOUNT-NEXT: shrl $4, %ecx
+; X86-NO-POPCOUNT-NEXT: andl $21845, %ecx # imm = 0x5555
+; X86-NO-POPCOUNT-NEXT: shrl $3, %eax
; X86-NO-POPCOUNT-NEXT: subl %ecx, %eax
; X86-NO-POPCOUNT-NEXT: movl %eax, %ecx
-; X86-NO-POPCOUNT-NEXT: andl $858993456, %ecx # imm = 0x33333330
+; X86-NO-POPCOUNT-NEXT: andl $858993459, %ecx # imm = 0x33333333
; X86-NO-POPCOUNT-NEXT: shrl $2, %eax
; X86-NO-POPCOUNT-NEXT: andl $858993459, %eax # imm = 0x33333333
; X86-NO-POPCOUNT-NEXT: addl %ecx, %eax
@@ -673,33 +674,34 @@ define i64 @ctpop_shifted_mask16(i64 %x) nounwind readnone {
; X86-NO-POPCOUNT-NEXT: shrl $4, %ecx
; X86-NO-POPCOUNT-NEXT: addl %eax, %ecx
; X86-NO-POPCOUNT-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NO-POPCOUNT-NEXT: imull $16843009, %ecx, %eax # imm = 0x1010101
-; X86-NO-POPCOUNT-NEXT: shrl $24, %eax
+; X86-NO-POPCOUNT-NEXT: movl %ecx, %eax
+; X86-NO-POPCOUNT-NEXT: shrl $8, %eax
+; X86-NO-POPCOUNT-NEXT: addl %ecx, %eax
+; X86-NO-POPCOUNT-NEXT: movzbl %al, %eax
; X86-NO-POPCOUNT-NEXT: xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT: retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask16:
; X64-NO-POPCOUNT: # %bb.0:
-; X64-NO-POPCOUNT-NEXT: movabsq $2251765453946880, %rax # imm = 0x7FFF800000000
-; X64-NO-POPCOUNT-NEXT: andq %rdi, %rax
-; X64-NO-POPCOUNT-NEXT: shrq %rdi
-; X64-NO-POPCOUNT-NEXT: movabsq $375294242324480, %rcx # imm = 0x1555400000000
-; X64-NO-POPCOUNT-NEXT: andq %rdi, %rcx
-; X64-NO-POPCOUNT-NEXT: subq %rcx, %rax
-; X64-NO-POPCOUNT-NEXT: movabsq $3689348800998014976, %rcx # imm = 0x3333333000000000
-; X64-NO-POPCOUNT-NEXT: andq %rax, %rcx
-; X64-NO-POPCOUNT-NEXT: shrq $2, %rax
-; X64-NO-POPCOUNT-NEXT: movabsq $3689348813882916864, %rdx # imm = 0x3333333300000000
-; X64-NO-POPCOUNT-NEXT: andq %rax, %rdx
-; X64-NO-POPCOUNT-NEXT: addq %rcx, %rdx
-; X64-NO-POPCOUNT-NEXT: movq %rdx, %rax
-; X64-NO-POPCOUNT-NEXT: shrq $4, %rax
-; X64-NO-POPCOUNT-NEXT: addq %rdx, %rax
-; X64-NO-POPCOUNT-NEXT: movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
-; X64-NO-POPCOUNT-NEXT: andq %rax, %rcx
-; X64-NO-POPCOUNT-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101
-; X64-NO-POPCOUNT-NEXT: imulq %rcx, %rax
-; X64-NO-POPCOUNT-NEXT: shrq $56, %rax
+; X64-NO-POPCOUNT-NEXT: shrq $35, %rdi
+; X64-NO-POPCOUNT-NEXT: movzwl %di, %eax
+; X64-NO-POPCOUNT-NEXT: movl %edi, %ecx
+; X64-NO-POPCOUNT-NEXT: shrl %ecx
+; X64-NO-POPCOUNT-NEXT: andl $21845, %ecx # imm = 0x5555
+; X64-NO-POPCOUNT-NEXT: subl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: movl %eax, %ecx
+; X64-NO-POPCOUNT-NEXT: andl $858993459, %ecx # imm = 0x33333333
+; X64-NO-POPCOUNT-NEXT: shrl $2, %eax
+; X64-NO-POPCOUNT-NEXT: andl $858993459, %eax # imm = 0x33333333
+; X64-NO-POPCOUNT-NEXT: addl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: movl %eax, %ecx
+; X64-NO-POPCOUNT-NEXT: shrl $4, %ecx
+; X64-NO-POPCOUNT-NEXT: addl %eax, %ecx
+; X64-NO-POPCOUNT-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
+; X64-NO-POPCOUNT-NEXT: movl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: shrl $8, %eax
+; X64-NO-POPCOUNT-NEXT: addl %ecx, %eax
+; X64-NO-POPCOUNT-NEXT: movzbl %al, %eax
; X64-NO-POPCOUNT-NEXT: retq
%mask = and i64 %x, 2251765453946880 ; 65535 << 35
%count = tail call i64 @llvm.ctpop.i64(i64 %mask)
>From d0f57f433117fa68fa4ee83b49321f9680a5910a Mon Sep 17 00:00:00 2001
From: milkHongYe <1483685464 at qq.com>
Date: Mon, 14 Sep 2026 17:54:18 +0800
Subject: [PATCH 3/3] remove unrelated range optimization
---
.../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 13 -------------
.../AMDGPU/bit-op-reduce-width-known-bits.ll | 6 +++---
2 files changed, 3 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index e4119d2b71839..710a1eda00cc1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12244,19 +12244,6 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
OutVal, SDNoFPClass);
}
- if (VT.isInteger()) {
- if (auto CR = Arg.getRange()) {
- if (!CR->isFullSet() && !CR->isEmptySet() &&
- !CR->isUpperWrapped()) {
- APInt Hi = CR->getUnsignedMax();
- unsigned Bits = std::max(Hi.getActiveBits(),
- (unsigned)IntegerType::MIN_INT_BITS);
- EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
- OutVal = DAG.getNode(ISD::AssertZext, dl, OutVal.getValueType(),
- OutVal, DAG.getValueType(SmallVT));
- }
- }
- }
ArgValues.push_back(OutVal);
}
diff --git a/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll b/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
index f7ede797d035d..c3f4d1d633376 100644
--- a/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
+++ b/llvm/test/CodeGen/AMDGPU/bit-op-reduce-width-known-bits.ll
@@ -11,7 +11,7 @@ define i64 @v_xor_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %
; CHECK-LABEL: v_xor_i64_known_hi_i32_from_arg_range:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
+; CHECK-NEXT: v_xor_b32_e32 v1, v1, v3
; CHECK-NEXT: v_xor_b32_e32 v0, v0, v2
; CHECK-NEXT: s_setpc_b64 s[30:31]
%xor = xor i64 %arg0, %arg1
@@ -24,7 +24,7 @@ define i64 @v_or_i64_known_hi_i32_from_arg_range(i64 range(i64 0, 4294967296) %a
; CHECK-LABEL: v_or_i64_known_hi_i32_from_arg_range:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
+; CHECK-NEXT: v_or_b32_e32 v1, v1, v3
; CHECK-NEXT: v_or_b32_e32 v0, v0, v2
; CHECK-NEXT: s_setpc_b64 s[30:31]
%or = or i64 %arg0, %arg1
@@ -50,7 +50,7 @@ define i64 @s_xor_i64_known_i32_from_arg_range(i64 range(i64 0, 65) inreg %arg)
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_not_b64 s[4:5], s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, s4
-; CHECK-NEXT: v_mov_b32_e32 v1, -1
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: s_setpc_b64 s[30:31]
%xor = xor i64 %arg, -1
ret i64 %xor
More information about the llvm-commits
mailing list