[llvm] [SDag][ARM][RISCV] Allow lowering CTPOP into a libcall (PR #101786)

Sergei Barannikov via llvm-commits llvm-commits at lists.llvm.org
Tue Aug 6 09:49:06 PDT 2024


https://github.com/s-barannikov updated https://github.com/llvm/llvm-project/pull/101786

>From e6b2fa6010a23a40cb4b226e2fe176ed7f318ce7 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88 at gmail.com>
Date: Fri, 2 Aug 2024 12:29:39 +0300
Subject: [PATCH 1/3] [SDag][ARM][RISCV] Allow lowering CTPOP into a libcall
 (#99752)

The main change is adding CTPOP to `RuntimeLibcalls.def` to allow
targets to use LibCall action for CTPOP. DAG legalizers are changed
accordingly.
---
 llvm/include/llvm/IR/RuntimeLibcalls.def      |    3 +
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |   38 +-
 .../SelectionDAG/LegalizeIntegerTypes.cpp     |   32 +-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |    5 +-
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |    3 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |    5 +-
 .../Analysis/CostModel/RISCV/int-bit-manip.ll |   16 +-
 llvm/test/CodeGen/ARM/popcnt.ll               |   64 +-
 llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll    | 1072 +++--------------
 .../CodeGen/RISCV/ctz_zero_return_test.ll     |   98 +-
 llvm/test/CodeGen/RISCV/pr56457.ll            |   31 +-
 llvm/test/CodeGen/RISCV/pr95271.ll            |   23 +-
 llvm/test/CodeGen/RISCV/rv32xtheadbb.ll       |   88 +-
 llvm/test/CodeGen/RISCV/rv32zbb.ll            |  311 ++---
 .../RISCV/rv64-legal-i32/rv64xtheadbb.ll      |   34 +-
 .../CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll   |   65 +-
 llvm/test/CodeGen/RISCV/rv64xtheadbb.ll       |   34 +-
 llvm/test/CodeGen/RISCV/rv64zbb.ll            |  126 +-
 llvm/test/CodeGen/RISCV/sextw-removal.ll      |   46 +-
 llvm/test/CodeGen/Thumb2/mve-ctpop.ll         |   62 +-
 20 files changed, 434 insertions(+), 1722 deletions(-)

diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 89aaf6d1ad83f..3dd75622b8e43 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -85,6 +85,9 @@ HANDLE_LIBCALL(NEG_I64, "__negdi2")
 HANDLE_LIBCALL(CTLZ_I32, "__clzsi2")
 HANDLE_LIBCALL(CTLZ_I64, "__clzdi2")
 HANDLE_LIBCALL(CTLZ_I128, "__clzti2")
+HANDLE_LIBCALL(CTPOP_I32, "__popcountsi2")
+HANDLE_LIBCALL(CTPOP_I64, "__popcountdi2")
+HANDLE_LIBCALL(CTPOP_I128, "__popcountti2")
 
 // Floating-point
 HANDLE_LIBCALL(ADD_F32, "__addsf3")
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index bdb7917073020..c91a2360d1599 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -140,12 +140,9 @@ class SelectionDAGLegalize {
                        RTLIB::Libcall Call_F128,
                        RTLIB::Libcall Call_PPCF128,
                        SmallVectorImpl<SDValue> &Results);
-  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
-                           RTLIB::Libcall Call_I8,
-                           RTLIB::Libcall Call_I16,
-                           RTLIB::Libcall Call_I32,
-                           RTLIB::Libcall Call_I64,
-                           RTLIB::Libcall Call_I128);
+  SDValue ExpandIntLibCall(SDNode *Node, bool IsSigned, RTLIB::Libcall Call_I8,
+                           RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
+                           RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
   void ExpandArgFPLibCall(SDNode *Node,
                           RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
                           RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
@@ -2209,7 +2206,7 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
   ExpandFPLibCall(Node, LC, Results);
 }
 
-SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
+SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode *Node, bool IsSigned,
                                                RTLIB::Libcall Call_I8,
                                                RTLIB::Libcall Call_I16,
                                                RTLIB::Libcall Call_I32,
@@ -2224,7 +2221,9 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
   case MVT::i64:  LC = Call_I64; break;
   case MVT::i128: LC = Call_I128; break;
   }
-  return ExpandLibCall(LC, Node, isSigned).first;
+  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
+         "LibCall explicitly requested, but not available");
+  return ExpandLibCall(LC, Node, IsSigned).first;
 }
 
 /// Expand the node to a libcall based on first argument type (for instance
@@ -5000,19 +4999,16 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
     break;
   case ISD::CTLZ_ZERO_UNDEF:
-    switch (Node->getSimpleValueType(0).SimpleTy) {
-    default:
-      llvm_unreachable("LibCall explicitly requested, but not available");
-    case MVT::i32:
-      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
-      break;
-    case MVT::i64:
-      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
-      break;
-    case MVT::i128:
-      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
-      break;
-    }
+    Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
+                                       RTLIB::UNKNOWN_LIBCALL,
+                                       RTLIB::UNKNOWN_LIBCALL, RTLIB::CTLZ_I32,
+                                       RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
+    break;
+  case ISD::CTPOP:
+    Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
+                                       RTLIB::UNKNOWN_LIBCALL,
+                                       RTLIB::UNKNOWN_LIBCALL, RTLIB::CTPOP_I32,
+                                       RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
     break;
   case ISD::RESET_FPENV: {
     // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index b1ada66aa9aeb..d00785025bac5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3850,15 +3850,33 @@ void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
   Hi = DAG.getConstant(0, dl, NVT);
 }
 
-void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
-                                          SDValue &Lo, SDValue &Hi) {
-  SDLoc dl(N);
+void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
+  SDValue Op = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  if (TLI.getOperationAction(ISD::CTPOP, VT) == TargetLoweringBase::LibCall) {
+    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+    if (VT == MVT::i32)
+      LC = RTLIB::CTPOP_I32;
+    else if (VT == MVT::i64)
+      LC = RTLIB::CTPOP_I64;
+    else if (VT == MVT::i128)
+      LC = RTLIB::CTPOP_I128;
+    assert(LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC) &&
+           "LibCall explicitly requested, but not available");
+    TargetLowering::MakeLibCallOptions CallOptions;
+    SDValue Res = TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, DL).first;
+    SplitInteger(Res, Lo, Hi);
+    return;
+  }
+
   // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
-  GetExpandedInteger(N->getOperand(0), Lo, Hi);
+  GetExpandedInteger(Op, Lo, Hi);
   EVT NVT = Lo.getValueType();
-  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
-                   DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
-  Hi = DAG.getConstant(0, dl, NVT);
+  Lo = DAG.getNode(ISD::ADD, DL, NVT, DAG.getNode(ISD::CTPOP, DL, NVT, Lo),
+                   DAG.getNode(ISD::CTPOP, DL, NVT, Hi));
+  Hi = DAG.getConstant(0, DL, NVT);
 }
 
 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7fa83a5999dfe..0989b79771c03 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9161,8 +9161,9 @@ SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
                         !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
     return SDValue();
 
-  // Emit Table Lookup if ISD::CTLZ and ISD::CTPOP are not legal.
-  if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) &&
+  // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
+  // to be expanded or converted to a libcall.
+  if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(ISD::CTPOP, VT) &&
       !isOperationLegal(ISD::CTLZ, VT))
     if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt))
       return V;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 75d16a42d0205..1d1ea22f6aac4 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -1204,7 +1204,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ROTR, VT, Expand);
   }
   setOperationAction(ISD::CTTZ,  MVT::i32, Custom);
-  setOperationAction(ISD::CTPOP, MVT::i32, Expand);
+  setOperationAction(ISD::CTPOP, MVT::i32, LibCall);
+  setOperationAction(ISD::CTPOP, MVT::i64, LibCall);
   if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
     setOperationAction(ISD::CTLZ, MVT::i32, Expand);
     setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, LibCall);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 9ee60b9db2837..4a2193c8d5328 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -393,7 +393,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
     }
   } else {
-    setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand);
+    setOperationAction(ISD::CTTZ, XLenVT, Expand);
+    if (!Subtarget.is64Bit())
+      setOperationAction(ISD::CTPOP, MVT::i32, LibCall);
+    setOperationAction(ISD::CTPOP, MVT::i64, LibCall);
     if (RV64LegalI32 && Subtarget.is64Bit())
       setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand);
   }
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
index 380f65b19b8fa..c0ecc63b82dca 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
@@ -159,7 +159,7 @@ define void @bitreverse() {
 
 define void @ctpop() {
 ; NOZVBB-LABEL: 'ctpop'
-; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
+; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %3 = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %4 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> undef)
@@ -169,7 +169,7 @@ define void @ctpop() {
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %8 = call <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %9 = call <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
-; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
+; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %12 = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %13 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %14 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> undef)
@@ -179,7 +179,7 @@ define void @ctpop() {
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
-; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
+; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %22 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %23 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
@@ -189,7 +189,7 @@ define void @ctpop() {
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
-; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
+; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %32 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
@@ -202,7 +202,7 @@ define void @ctpop() {
 ; NOZVBB-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; ZVBB-LABEL: 'ctpop'
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> undef)
@@ -212,7 +212,7 @@ define void @ctpop() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> undef)
@@ -222,7 +222,7 @@ define void @ctpop() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
@@ -232,7 +232,7 @@ define void @ctpop() {
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
-; ZVBB-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
+; ZVBB-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
 ; ZVBB-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
diff --git a/llvm/test/CodeGen/ARM/popcnt.ll b/llvm/test/CodeGen/ARM/popcnt.ll
index edcae5e141e73..6c01c516be9d2 100644
--- a/llvm/test/CodeGen/ARM/popcnt.ll
+++ b/llvm/test/CodeGen/ARM/popcnt.ll
@@ -324,30 +324,7 @@ define i32 @ctpop16(i16 %x) nounwind readnone {
 define i32 @ctpop32(i32 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    ldr r1, .LCPI22_0
-; CHECK-NEXT:    ldr r2, .LCPI22_3
-; CHECK-NEXT:    and r1, r1, r0, lsr #1
-; CHECK-NEXT:    ldr r12, .LCPI22_1
-; CHECK-NEXT:    sub r0, r0, r1
-; CHECK-NEXT:    ldr r3, .LCPI22_2
-; CHECK-NEXT:    and r1, r0, r2
-; CHECK-NEXT:    and r0, r2, r0, lsr #2
-; CHECK-NEXT:    add r0, r1, r0
-; CHECK-NEXT:    add r0, r0, r0, lsr #4
-; CHECK-NEXT:    and r0, r0, r12
-; CHECK-NEXT:    mul r1, r0, r3
-; CHECK-NEXT:    lsr r0, r1, #24
-; CHECK-NEXT:    mov pc, lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI22_0:
-; CHECK-NEXT:    .long 1431655765 @ 0x55555555
-; CHECK-NEXT:  .LCPI22_1:
-; CHECK-NEXT:    .long 252645135 @ 0xf0f0f0f
-; CHECK-NEXT:  .LCPI22_2:
-; CHECK-NEXT:    .long 16843009 @ 0x1010101
-; CHECK-NEXT:  .LCPI22_3:
-; CHECK-NEXT:    .long 858993459 @ 0x33333333
+; CHECK-NEXT:    b __popcountsi2
   %count = tail call i32 @llvm.ctpop.i32(i32 %x)
   ret i32 %count
 }
@@ -355,42 +332,11 @@ define i32 @ctpop32(i32 %x) nounwind readnone {
 define i32 @ctpop64(i64 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop64:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    ldr r2, .LCPI23_0
-; CHECK-NEXT:    ldr r3, .LCPI23_3
-; CHECK-NEXT:    and r4, r2, r0, lsr #1
-; CHECK-NEXT:    and r2, r2, r1, lsr #1
-; CHECK-NEXT:    sub r0, r0, r4
-; CHECK-NEXT:    sub r1, r1, r2
-; CHECK-NEXT:    and r4, r0, r3
-; CHECK-NEXT:    and r2, r1, r3
-; CHECK-NEXT:    and r0, r3, r0, lsr #2
-; CHECK-NEXT:    and r1, r3, r1, lsr #2
-; CHECK-NEXT:    add r0, r4, r0
-; CHECK-NEXT:    ldr lr, .LCPI23_1
-; CHECK-NEXT:    add r1, r2, r1
-; CHECK-NEXT:    ldr r12, .LCPI23_2
-; CHECK-NEXT:    add r0, r0, r0, lsr #4
-; CHECK-NEXT:    and r0, r0, lr
-; CHECK-NEXT:    add r1, r1, r1, lsr #4
-; CHECK-NEXT:    mul r2, r0, r12
-; CHECK-NEXT:    and r0, r1, lr
-; CHECK-NEXT:    mul r1, r0, r12
-; CHECK-NEXT:    lsr r0, r2, #24
-; CHECK-NEXT:    add r0, r0, r1, lsr #24
-; CHECK-NEXT:    pop {r4, lr}
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    bl __popcountdi2
+; CHECK-NEXT:    pop {r11, lr}
 ; CHECK-NEXT:    mov pc, lr
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI23_0:
-; CHECK-NEXT:    .long 1431655765 @ 0x55555555
-; CHECK-NEXT:  .LCPI23_1:
-; CHECK-NEXT:    .long 252645135 @ 0xf0f0f0f
-; CHECK-NEXT:  .LCPI23_2:
-; CHECK-NEXT:    .long 16843009 @ 0x1010101
-; CHECK-NEXT:  .LCPI23_3:
-; CHECK-NEXT:    .long 858993459 @ 0x33333333
   %count = tail call i64 @llvm.ctpop.i64(i64 %x)
   %conv = trunc i64 %count to i32
   ret i32 %conv
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 8caa64c9572ce..9123017918094 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -1156,46 +1156,30 @@ define i16 @test_ctlz_i16(i16 %a) nounwind {
 }
 
 define i32 @test_ctlz_i32(i32 %a) nounwind {
-; RV32I-LABEL: test_ctlz_i32:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    beqz a0, .LBB10_2
-; RV32I-NEXT:  # %bb.1: # %cond.false
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB10_2:
-; RV32I-NEXT:    li a0, 32
-; RV32I-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctlz_i32:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    beqz a0, .LBB10_2
+; RV32_NOZBB-NEXT:  # %bb.1: # %cond.false
+; RV32_NOZBB-NEXT:    addi sp, sp, -16
+; RV32_NOZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    srli a1, a0, 1
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    call __popcountsi2
+; RV32_NOZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    addi sp, sp, 16
+; RV32_NOZBB-NEXT:    ret
+; RV32_NOZBB-NEXT:  .LBB10_2:
+; RV32_NOZBB-NEXT:    li a0, 32
+; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64I-LABEL: test_ctlz_i32:
 ; RV64I:       # %bb.0:
@@ -1239,46 +1223,6 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    li a0, 32
 ; RV64I-NEXT:    ret
 ;
-; RV32M-LABEL: test_ctlz_i32:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    beqz a0, .LBB10_2
-; RV32M-NEXT:  # %bb.1: # %cond.false
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    lui a2, 349525
-; RV32M-NEXT:    addi a2, a2, 1365
-; RV32M-NEXT:    and a1, a1, a2
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    lui a1, 209715
-; RV32M-NEXT:    addi a1, a1, 819
-; RV32M-NEXT:    and a2, a0, a1
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    add a0, a2, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    lui a1, 61681
-; RV32M-NEXT:    addi a1, a1, -241
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    lui a1, 4112
-; RV32M-NEXT:    addi a1, a1, 257
-; RV32M-NEXT:    mul a0, a0, a1
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    ret
-; RV32M-NEXT:  .LBB10_2:
-; RV32M-NEXT:    li a0, 32
-; RV32M-NEXT:    ret
-;
 ; RV64M-LABEL: test_ctlz_i32:
 ; RV64M:       # %bb.0:
 ; RV64M-NEXT:    sext.w a1, a0
@@ -1346,240 +1290,75 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 }
 
 define i64 @test_ctlz_i64(i64 %a) nounwind {
-; RV32I-LABEL: test_ctlz_i64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a4, a2, 1365
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a3, a2, 819
-; RV32I-NEXT:    lui a2, 61681
-; RV32I-NEXT:    addi a2, a2, -241
-; RV32I-NEXT:    bnez a1, .LBB11_2
-; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    addi a0, a0, 32
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB11_2:
-; RV32I-NEXT:    srli a0, a1, 1
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: test_ctlz_i64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    beqz a0, .LBB11_2
-; RV64I-NEXT:  # %bb.1: # %cond.false
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 8
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB11_2:
-; RV64I-NEXT:    li a0, 64
-; RV64I-NEXT:    ret
-;
-; RV32M-LABEL: test_ctlz_i64:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    lui a2, 349525
-; RV32M-NEXT:    addi a5, a2, 1365
-; RV32M-NEXT:    lui a2, 209715
-; RV32M-NEXT:    addi a4, a2, 819
-; RV32M-NEXT:    lui a2, 61681
-; RV32M-NEXT:    addi a2, a2, -241
-; RV32M-NEXT:    lui a3, 4112
-; RV32M-NEXT:    addi a3, a3, 257
-; RV32M-NEXT:    bnez a1, .LBB11_2
-; RV32M-NEXT:  # %bb.1:
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    and a1, a1, a5
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    and a1, a0, a4
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a4
-; RV32M-NEXT:    add a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    and a0, a0, a2
-; RV32M-NEXT:    mul a0, a0, a3
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    addi a0, a0, 32
-; RV32M-NEXT:    li a1, 0
-; RV32M-NEXT:    ret
-; RV32M-NEXT:  .LBB11_2:
-; RV32M-NEXT:    srli a0, a1, 1
-; RV32M-NEXT:    or a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    and a1, a1, a5
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    and a1, a0, a4
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a4
-; RV32M-NEXT:    add a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    and a0, a0, a2
-; RV32M-NEXT:    mul a0, a0, a3
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    li a1, 0
-; RV32M-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctlz_i64:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    addi sp, sp, -16
+; RV32_NOZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    mv s1, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 1
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    call __popcountsi2
+; RV32_NOZBB-NEXT:    mv s0, a0
+; RV32_NOZBB-NEXT:    srli a0, s1, 1
+; RV32_NOZBB-NEXT:    or a0, s1, a0
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    call __popcountsi2
+; RV32_NOZBB-NEXT:    bnez s1, .LBB11_2
+; RV32_NOZBB-NEXT:  # %bb.1:
+; RV32_NOZBB-NEXT:    addi a0, s0, 32
+; RV32_NOZBB-NEXT:  .LBB11_2:
+; RV32_NOZBB-NEXT:    li a1, 0
+; RV32_NOZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    addi sp, sp, 16
+; RV32_NOZBB-NEXT:    ret
 ;
-; RV64M-LABEL: test_ctlz_i64:
-; RV64M:       # %bb.0:
-; RV64M-NEXT:    beqz a0, .LBB11_2
-; RV64M-NEXT:  # %bb.1: # %cond.false
-; RV64M-NEXT:    srli a1, a0, 1
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 2
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 4
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 8
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 16
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 32
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    not a0, a0
-; RV64M-NEXT:    srli a1, a0, 1
-; RV64M-NEXT:    lui a2, 349525
-; RV64M-NEXT:    addiw a2, a2, 1365
-; RV64M-NEXT:    slli a3, a2, 32
-; RV64M-NEXT:    add a2, a2, a3
-; RV64M-NEXT:    and a1, a1, a2
-; RV64M-NEXT:    sub a0, a0, a1
-; RV64M-NEXT:    lui a1, 209715
-; RV64M-NEXT:    addiw a1, a1, 819
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a2, a0, a1
-; RV64M-NEXT:    srli a0, a0, 2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    add a0, a2, a0
-; RV64M-NEXT:    srli a1, a0, 4
-; RV64M-NEXT:    add a0, a0, a1
-; RV64M-NEXT:    lui a1, 61681
-; RV64M-NEXT:    addiw a1, a1, -241
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    lui a1, 4112
-; RV64M-NEXT:    addiw a1, a1, 257
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    mul a0, a0, a1
-; RV64M-NEXT:    srli a0, a0, 56
-; RV64M-NEXT:    ret
-; RV64M-NEXT:  .LBB11_2:
-; RV64M-NEXT:    li a0, 64
-; RV64M-NEXT:    ret
+; RV64NOZBB-LABEL: test_ctlz_i64:
+; RV64NOZBB:       # %bb.0:
+; RV64NOZBB-NEXT:    beqz a0, .LBB11_2
+; RV64NOZBB-NEXT:  # %bb.1: # %cond.false
+; RV64NOZBB-NEXT:    addi sp, sp, -16
+; RV64NOZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    srli a1, a0, 1
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 2
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 4
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 8
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 16
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 32
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    not a0, a0
+; RV64NOZBB-NEXT:    call __popcountdi2
+; RV64NOZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    addi sp, sp, 16
+; RV64NOZBB-NEXT:    ret
+; RV64NOZBB-NEXT:  .LBB11_2:
+; RV64NOZBB-NEXT:    li a0, 64
+; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctlz_i64:
 ; RV32ZBB:       # %bb.0:
@@ -1793,41 +1572,20 @@ define i16 @test_ctlz_i16_zero_undef(i16 %a) nounwind {
 }
 
 define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
-; RV32I-LABEL: test_ctlz_i32_zero_undef:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctlz_i32_zero_undef:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    srli a1, a0, 1
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    tail __popcountsi2
 ;
 ; RV64I-LABEL: test_ctlz_i32_zero_undef:
 ; RV64I:       # %bb.0:
@@ -1865,41 +1623,6 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ret
 ;
-; RV32M-LABEL: test_ctlz_i32_zero_undef:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    lui a2, 349525
-; RV32M-NEXT:    addi a2, a2, 1365
-; RV32M-NEXT:    and a1, a1, a2
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    lui a1, 209715
-; RV32M-NEXT:    addi a1, a1, 819
-; RV32M-NEXT:    and a2, a0, a1
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    add a0, a2, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    lui a1, 61681
-; RV32M-NEXT:    addi a1, a1, -241
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    lui a1, 4112
-; RV32M-NEXT:    addi a1, a1, 257
-; RV32M-NEXT:    mul a0, a0, a1
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    ret
-;
 ; RV64M-LABEL: test_ctlz_i32_zero_undef:
 ; RV64M:       # %bb.0:
 ; RV64M-NEXT:    srliw a1, a0, 1
@@ -1961,230 +1684,65 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
 }
 
 define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
-; RV32I-LABEL: test_ctlz_i64_zero_undef:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a4, a2, 1365
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a3, a2, 819
-; RV32I-NEXT:    lui a2, 61681
-; RV32I-NEXT:    addi a2, a2, -241
-; RV32I-NEXT:    bnez a1, .LBB15_2
-; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    addi a0, a0, 32
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB15_2:
-; RV32I-NEXT:    srli a0, a1, 1
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 8
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    srli a1, a0, 16
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: test_ctlz_i64_zero_undef:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 8
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 16
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    srli a1, a0, 32
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ret
-;
-; RV32M-LABEL: test_ctlz_i64_zero_undef:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    lui a2, 349525
-; RV32M-NEXT:    addi a5, a2, 1365
-; RV32M-NEXT:    lui a2, 209715
-; RV32M-NEXT:    addi a4, a2, 819
-; RV32M-NEXT:    lui a2, 61681
-; RV32M-NEXT:    addi a2, a2, -241
-; RV32M-NEXT:    lui a3, 4112
-; RV32M-NEXT:    addi a3, a3, 257
-; RV32M-NEXT:    bnez a1, .LBB15_2
-; RV32M-NEXT:  # %bb.1:
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    and a1, a1, a5
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    and a1, a0, a4
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a4
-; RV32M-NEXT:    add a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    and a0, a0, a2
-; RV32M-NEXT:    mul a0, a0, a3
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    addi a0, a0, 32
-; RV32M-NEXT:    li a1, 0
-; RV32M-NEXT:    ret
-; RV32M-NEXT:  .LBB15_2:
-; RV32M-NEXT:    srli a0, a1, 1
-; RV32M-NEXT:    or a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 2
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 8
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    srli a1, a0, 16
-; RV32M-NEXT:    or a0, a0, a1
-; RV32M-NEXT:    not a0, a0
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    and a1, a1, a5
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    and a1, a0, a4
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a4
-; RV32M-NEXT:    add a0, a1, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    and a0, a0, a2
-; RV32M-NEXT:    mul a0, a0, a3
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    li a1, 0
-; RV32M-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctlz_i64_zero_undef:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    addi sp, sp, -16
+; RV32_NOZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    mv s1, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 1
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    call __popcountsi2
+; RV32_NOZBB-NEXT:    mv s0, a0
+; RV32_NOZBB-NEXT:    srli a0, s1, 1
+; RV32_NOZBB-NEXT:    or a0, s1, a0
+; RV32_NOZBB-NEXT:    srli a1, a0, 2
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 4
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 8
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    srli a1, a0, 16
+; RV32_NOZBB-NEXT:    or a0, a0, a1
+; RV32_NOZBB-NEXT:    not a0, a0
+; RV32_NOZBB-NEXT:    call __popcountsi2
+; RV32_NOZBB-NEXT:    bnez s1, .LBB15_2
+; RV32_NOZBB-NEXT:  # %bb.1:
+; RV32_NOZBB-NEXT:    addi a0, s0, 32
+; RV32_NOZBB-NEXT:  .LBB15_2:
+; RV32_NOZBB-NEXT:    li a1, 0
+; RV32_NOZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    addi sp, sp, 16
+; RV32_NOZBB-NEXT:    ret
 ;
-; RV64M-LABEL: test_ctlz_i64_zero_undef:
-; RV64M:       # %bb.0:
-; RV64M-NEXT:    srli a1, a0, 1
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 2
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 4
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 8
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 16
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    srli a1, a0, 32
-; RV64M-NEXT:    or a0, a0, a1
-; RV64M-NEXT:    not a0, a0
-; RV64M-NEXT:    srli a1, a0, 1
-; RV64M-NEXT:    lui a2, 349525
-; RV64M-NEXT:    addiw a2, a2, 1365
-; RV64M-NEXT:    slli a3, a2, 32
-; RV64M-NEXT:    add a2, a2, a3
-; RV64M-NEXT:    and a1, a1, a2
-; RV64M-NEXT:    sub a0, a0, a1
-; RV64M-NEXT:    lui a1, 209715
-; RV64M-NEXT:    addiw a1, a1, 819
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a2, a0, a1
-; RV64M-NEXT:    srli a0, a0, 2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    add a0, a2, a0
-; RV64M-NEXT:    srli a1, a0, 4
-; RV64M-NEXT:    add a0, a0, a1
-; RV64M-NEXT:    lui a1, 61681
-; RV64M-NEXT:    addiw a1, a1, -241
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    lui a1, 4112
-; RV64M-NEXT:    addiw a1, a1, 257
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    mul a0, a0, a1
-; RV64M-NEXT:    srli a0, a0, 56
-; RV64M-NEXT:    ret
+; RV64NOZBB-LABEL: test_ctlz_i64_zero_undef:
+; RV64NOZBB:       # %bb.0:
+; RV64NOZBB-NEXT:    srli a1, a0, 1
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 2
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 4
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 8
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 16
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    srli a1, a0, 32
+; RV64NOZBB-NEXT:    or a0, a0, a1
+; RV64NOZBB-NEXT:    not a0, a0
+; RV64NOZBB-NEXT:    tail __popcountdi2
 ;
 ; RV32ZBB-LABEL: test_ctlz_i64_zero_undef:
 ; RV32ZBB:       # %bb.0:
@@ -2398,30 +1956,9 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 }
 
 define i32 @test_ctpop_i32(i32 %a) nounwind {
-; RV32I-LABEL: test_ctpop_i32:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctpop_i32:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    tail __popcountsi2
 ;
 ; RV64I-LABEL: test_ctpop_i32:
 ; RV64I:       # %bb.0:
@@ -2448,30 +1985,6 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srliw a0, a0, 24
 ; RV64I-NEXT:    ret
 ;
-; RV32M-LABEL: test_ctpop_i32:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    srli a1, a0, 1
-; RV32M-NEXT:    lui a2, 349525
-; RV32M-NEXT:    addi a2, a2, 1365
-; RV32M-NEXT:    and a1, a1, a2
-; RV32M-NEXT:    sub a0, a0, a1
-; RV32M-NEXT:    lui a1, 209715
-; RV32M-NEXT:    addi a1, a1, 819
-; RV32M-NEXT:    and a2, a0, a1
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    add a0, a2, a0
-; RV32M-NEXT:    srli a1, a0, 4
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    lui a1, 61681
-; RV32M-NEXT:    addi a1, a1, -241
-; RV32M-NEXT:    and a0, a0, a1
-; RV32M-NEXT:    lui a1, 4112
-; RV32M-NEXT:    addi a1, a1, 257
-; RV32M-NEXT:    mul a0, a0, a1
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    ret
-;
 ; RV64M-LABEL: test_ctpop_i32:
 ; RV64M:       # %bb.0:
 ; RV64M-NEXT:    srli a1, a0, 1
@@ -2508,28 +2021,7 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ;
 ; RV32XTHEADBB-LABEL: test_ctpop_i32:
 ; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    srli a1, a0, 1
-; RV32XTHEADBB-NEXT:    lui a2, 349525
-; RV32XTHEADBB-NEXT:    addi a2, a2, 1365
-; RV32XTHEADBB-NEXT:    and a1, a1, a2
-; RV32XTHEADBB-NEXT:    sub a0, a0, a1
-; RV32XTHEADBB-NEXT:    lui a1, 209715
-; RV32XTHEADBB-NEXT:    addi a1, a1, 819
-; RV32XTHEADBB-NEXT:    and a2, a0, a1
-; RV32XTHEADBB-NEXT:    srli a0, a0, 2
-; RV32XTHEADBB-NEXT:    and a0, a0, a1
-; RV32XTHEADBB-NEXT:    add a0, a2, a0
-; RV32XTHEADBB-NEXT:    srli a1, a0, 4
-; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    lui a1, 61681
-; RV32XTHEADBB-NEXT:    addi a1, a1, -241
-; RV32XTHEADBB-NEXT:    and a0, a0, a1
-; RV32XTHEADBB-NEXT:    slli a1, a0, 8
-; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    slli a1, a0, 16
-; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    srli a0, a0, 24
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NEXT:    tail __popcountsi2
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i32:
 ; RV64XTHEADBB:       # %bb.0:
@@ -2560,150 +2052,18 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 }
 
 define i64 @test_ctpop_i64(i64 %a) nounwind {
-; RV32I-LABEL: test_ctpop_i64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a2, a1, 1
-; RV32I-NEXT:    lui a3, 349525
-; RV32I-NEXT:    addi a3, a3, 1365
-; RV32I-NEXT:    and a2, a2, a3
-; RV32I-NEXT:    sub a1, a1, a2
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a2, a2, 819
-; RV32I-NEXT:    and a4, a1, a2
-; RV32I-NEXT:    srli a1, a1, 2
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    add a1, a4, a1
-; RV32I-NEXT:    srli a4, a1, 4
-; RV32I-NEXT:    add a1, a1, a4
-; RV32I-NEXT:    lui a4, 61681
-; RV32I-NEXT:    addi a4, a4, -241
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    slli a5, a1, 8
-; RV32I-NEXT:    add a1, a1, a5
-; RV32I-NEXT:    slli a5, a1, 16
-; RV32I-NEXT:    add a1, a1, a5
-; RV32I-NEXT:    srli a1, a1, 24
-; RV32I-NEXT:    srli a5, a0, 1
-; RV32I-NEXT:    and a3, a5, a3
-; RV32I-NEXT:    sub a0, a0, a3
-; RV32I-NEXT:    and a3, a0, a2
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    add a0, a3, a0
-; RV32I-NEXT:    srli a2, a0, 4
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    and a0, a0, a4
-; RV32I-NEXT:    slli a2, a0, 8
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    slli a2, a0, 16
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: test_ctpop_i64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ret
-;
-; RV32M-LABEL: test_ctpop_i64:
-; RV32M:       # %bb.0:
-; RV32M-NEXT:    srli a2, a1, 1
-; RV32M-NEXT:    lui a3, 349525
-; RV32M-NEXT:    addi a3, a3, 1365
-; RV32M-NEXT:    and a2, a2, a3
-; RV32M-NEXT:    sub a1, a1, a2
-; RV32M-NEXT:    lui a2, 209715
-; RV32M-NEXT:    addi a2, a2, 819
-; RV32M-NEXT:    and a4, a1, a2
-; RV32M-NEXT:    srli a1, a1, 2
-; RV32M-NEXT:    and a1, a1, a2
-; RV32M-NEXT:    add a1, a4, a1
-; RV32M-NEXT:    srli a4, a1, 4
-; RV32M-NEXT:    add a1, a1, a4
-; RV32M-NEXT:    lui a4, 61681
-; RV32M-NEXT:    addi a4, a4, -241
-; RV32M-NEXT:    and a1, a1, a4
-; RV32M-NEXT:    lui a5, 4112
-; RV32M-NEXT:    addi a5, a5, 257
-; RV32M-NEXT:    mul a1, a1, a5
-; RV32M-NEXT:    srli a1, a1, 24
-; RV32M-NEXT:    srli a6, a0, 1
-; RV32M-NEXT:    and a3, a6, a3
-; RV32M-NEXT:    sub a0, a0, a3
-; RV32M-NEXT:    and a3, a0, a2
-; RV32M-NEXT:    srli a0, a0, 2
-; RV32M-NEXT:    and a0, a0, a2
-; RV32M-NEXT:    add a0, a3, a0
-; RV32M-NEXT:    srli a2, a0, 4
-; RV32M-NEXT:    add a0, a0, a2
-; RV32M-NEXT:    and a0, a0, a4
-; RV32M-NEXT:    mul a0, a0, a5
-; RV32M-NEXT:    srli a0, a0, 24
-; RV32M-NEXT:    add a0, a0, a1
-; RV32M-NEXT:    li a1, 0
-; RV32M-NEXT:    ret
+; RV32_NOZBB-LABEL: test_ctpop_i64:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    addi sp, sp, -16
+; RV32_NOZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    call __popcountdi2
+; RV32_NOZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    addi sp, sp, 16
+; RV32_NOZBB-NEXT:    ret
 ;
-; RV64M-LABEL: test_ctpop_i64:
-; RV64M:       # %bb.0:
-; RV64M-NEXT:    srli a1, a0, 1
-; RV64M-NEXT:    lui a2, 349525
-; RV64M-NEXT:    addiw a2, a2, 1365
-; RV64M-NEXT:    slli a3, a2, 32
-; RV64M-NEXT:    add a2, a2, a3
-; RV64M-NEXT:    and a1, a1, a2
-; RV64M-NEXT:    sub a0, a0, a1
-; RV64M-NEXT:    lui a1, 209715
-; RV64M-NEXT:    addiw a1, a1, 819
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a2, a0, a1
-; RV64M-NEXT:    srli a0, a0, 2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    add a0, a2, a0
-; RV64M-NEXT:    srli a1, a0, 4
-; RV64M-NEXT:    add a0, a0, a1
-; RV64M-NEXT:    lui a1, 61681
-; RV64M-NEXT:    addiw a1, a1, -241
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    and a0, a0, a1
-; RV64M-NEXT:    lui a1, 4112
-; RV64M-NEXT:    addiw a1, a1, 257
-; RV64M-NEXT:    slli a2, a1, 32
-; RV64M-NEXT:    add a1, a1, a2
-; RV64M-NEXT:    mul a0, a0, a1
-; RV64M-NEXT:    srli a0, a0, 56
-; RV64M-NEXT:    ret
+; RV64NOZBB-LABEL: test_ctpop_i64:
+; RV64NOZBB:       # %bb.0:
+; RV64NOZBB-NEXT:    tail __popcountdi2
 ;
 ; RV32ZBB-LABEL: test_ctpop_i64:
 ; RV32ZBB:       # %bb.0:
@@ -2720,78 +2080,16 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ;
 ; RV32XTHEADBB-LABEL: test_ctpop_i64:
 ; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    srli a2, a1, 1
-; RV32XTHEADBB-NEXT:    lui a3, 349525
-; RV32XTHEADBB-NEXT:    addi a3, a3, 1365
-; RV32XTHEADBB-NEXT:    and a2, a2, a3
-; RV32XTHEADBB-NEXT:    sub a1, a1, a2
-; RV32XTHEADBB-NEXT:    lui a2, 209715
-; RV32XTHEADBB-NEXT:    addi a2, a2, 819
-; RV32XTHEADBB-NEXT:    and a4, a1, a2
-; RV32XTHEADBB-NEXT:    srli a1, a1, 2
-; RV32XTHEADBB-NEXT:    and a1, a1, a2
-; RV32XTHEADBB-NEXT:    add a1, a4, a1
-; RV32XTHEADBB-NEXT:    srli a4, a1, 4
-; RV32XTHEADBB-NEXT:    add a1, a1, a4
-; RV32XTHEADBB-NEXT:    lui a4, 61681
-; RV32XTHEADBB-NEXT:    addi a4, a4, -241
-; RV32XTHEADBB-NEXT:    and a1, a1, a4
-; RV32XTHEADBB-NEXT:    slli a5, a1, 8
-; RV32XTHEADBB-NEXT:    add a1, a1, a5
-; RV32XTHEADBB-NEXT:    slli a5, a1, 16
-; RV32XTHEADBB-NEXT:    add a1, a1, a5
-; RV32XTHEADBB-NEXT:    srli a1, a1, 24
-; RV32XTHEADBB-NEXT:    srli a5, a0, 1
-; RV32XTHEADBB-NEXT:    and a3, a5, a3
-; RV32XTHEADBB-NEXT:    sub a0, a0, a3
-; RV32XTHEADBB-NEXT:    and a3, a0, a2
-; RV32XTHEADBB-NEXT:    srli a0, a0, 2
-; RV32XTHEADBB-NEXT:    and a0, a0, a2
-; RV32XTHEADBB-NEXT:    add a0, a3, a0
-; RV32XTHEADBB-NEXT:    srli a2, a0, 4
-; RV32XTHEADBB-NEXT:    add a0, a0, a2
-; RV32XTHEADBB-NEXT:    and a0, a0, a4
-; RV32XTHEADBB-NEXT:    slli a2, a0, 8
-; RV32XTHEADBB-NEXT:    add a0, a0, a2
-; RV32XTHEADBB-NEXT:    slli a2, a0, 16
-; RV32XTHEADBB-NEXT:    add a0, a0, a2
-; RV32XTHEADBB-NEXT:    srli a0, a0, 24
-; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    li a1, 0
+; RV32XTHEADBB-NEXT:    addi sp, sp, -16
+; RV32XTHEADBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    call __popcountdi2
+; RV32XTHEADBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    addi sp, sp, 16
 ; RV32XTHEADBB-NEXT:    ret
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i64:
 ; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    srli a1, a0, 1
-; RV64XTHEADBB-NEXT:    lui a2, 349525
-; RV64XTHEADBB-NEXT:    addiw a2, a2, 1365
-; RV64XTHEADBB-NEXT:    slli a3, a2, 32
-; RV64XTHEADBB-NEXT:    add a2, a2, a3
-; RV64XTHEADBB-NEXT:    and a1, a1, a2
-; RV64XTHEADBB-NEXT:    sub a0, a0, a1
-; RV64XTHEADBB-NEXT:    lui a1, 209715
-; RV64XTHEADBB-NEXT:    addiw a1, a1, 819
-; RV64XTHEADBB-NEXT:    slli a2, a1, 32
-; RV64XTHEADBB-NEXT:    add a1, a1, a2
-; RV64XTHEADBB-NEXT:    and a2, a0, a1
-; RV64XTHEADBB-NEXT:    srli a0, a0, 2
-; RV64XTHEADBB-NEXT:    and a0, a0, a1
-; RV64XTHEADBB-NEXT:    add a0, a2, a0
-; RV64XTHEADBB-NEXT:    srli a1, a0, 4
-; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    lui a1, 61681
-; RV64XTHEADBB-NEXT:    addiw a1, a1, -241
-; RV64XTHEADBB-NEXT:    slli a2, a1, 32
-; RV64XTHEADBB-NEXT:    add a1, a1, a2
-; RV64XTHEADBB-NEXT:    and a0, a0, a1
-; RV64XTHEADBB-NEXT:    slli a1, a0, 8
-; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    slli a1, a0, 16
-; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    slli a1, a0, 32
-; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    srli a0, a0, 56
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NEXT:    tail __popcountdi2
   %1 = call i64 @llvm.ctpop.i64(i64 %a)
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
index fe6e20d852d59..d57c4d653b2ae 100644
--- a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
+++ b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
@@ -602,14 +602,11 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ;
 ; RV32I-LABEL: ctlz:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a4, a2, 1365
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a3, a2, 819
-; RV32I-NEXT:    lui a2, 61681
-; RV32I-NEXT:    addi a2, a2, -241
-; RV32I-NEXT:    bnez a1, .LBB7_2
-; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 2
@@ -621,27 +618,10 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    addi a0, a0, 32
-; RV32I-NEXT:    andi a0, a0, 63
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB7_2:
-; RV32I-NEXT:    srli a0, a1, 1
-; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    srli a0, s1, 1
+; RV32I-NEXT:    or a0, s1, a0
 ; RV32I-NEXT:    srli a1, a0, 2
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 4
@@ -651,26 +631,22 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    bnez s1, .LBB7_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    addi a0, s0, 32
+; RV32I-NEXT:  .LBB7_2: # %entry
 ; RV32I-NEXT:    andi a0, a0, 63
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: ctlz:
 ; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -684,36 +660,10 @@ define signext i32 @ctlz(i64 %b) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a0, a0, 2
-; RV64I-NEXT:    srli a0, a0, 58
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    andi a0, a0, 63
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 
 
diff --git a/llvm/test/CodeGen/RISCV/pr56457.ll b/llvm/test/CodeGen/RISCV/pr56457.ll
index ba08aa838bf99..19cc8b3af208f 100644
--- a/llvm/test/CodeGen/RISCV/pr56457.ll
+++ b/llvm/test/CodeGen/RISCV/pr56457.ll
@@ -9,6 +9,8 @@ define i15 @foo(i15 %x) nounwind {
 ; CHECK-NEXT:    slli a1, a0, 49
 ; CHECK-NEXT:    beqz a1, .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %cond.false
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    srli a1, a1, 50
 ; CHECK-NEXT:    or a0, a0, a1
 ; CHECK-NEXT:    slli a1, a0, 49
@@ -21,34 +23,11 @@ define i15 @foo(i15 %x) nounwind {
 ; CHECK-NEXT:    srli a1, a1, 57
 ; CHECK-NEXT:    or a0, a0, a1
 ; CHECK-NEXT:    not a0, a0
-; CHECK-NEXT:    srli a1, a0, 1
-; CHECK-NEXT:    lui a2, 1
-; CHECK-NEXT:    addiw a2, a2, 1365
-; CHECK-NEXT:    and a1, a1, a2
 ; CHECK-NEXT:    slli a0, a0, 49
 ; CHECK-NEXT:    srli a0, a0, 49
-; CHECK-NEXT:    sub a0, a0, a1
-; CHECK-NEXT:    lui a1, 209715
-; CHECK-NEXT:    addiw a1, a1, 819
-; CHECK-NEXT:    slli a2, a1, 32
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    and a2, a0, a1
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    and a0, a0, a1
-; CHECK-NEXT:    add a0, a2, a0
-; CHECK-NEXT:    srli a1, a0, 4
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    lui a1, 61681
-; CHECK-NEXT:    addiw a1, a1, -241
-; CHECK-NEXT:    slli a2, a1, 32
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    and a0, a0, a1
-; CHECK-NEXT:    lui a1, 4112
-; CHECK-NEXT:    addiw a1, a1, 257
-; CHECK-NEXT:    slli a2, a1, 32
-; CHECK-NEXT:    add a1, a1, a2
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    srli a0, a0, 56
+; CHECK-NEXT:    call __popcountdi2
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    li a0, 15
diff --git a/llvm/test/CodeGen/RISCV/pr95271.ll b/llvm/test/CodeGen/RISCV/pr95271.ll
index 950e6fb5f37ec..e9a5d2d8de750 100644
--- a/llvm/test/CodeGen/RISCV/pr95271.ll
+++ b/llvm/test/CodeGen/RISCV/pr95271.ll
@@ -7,28 +7,7 @@ define i32 @PR95271(ptr %p) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a0, 0(a0)
 ; RV32I-NEXT:    addi a0, a0, 1
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    ret
+; RV32I-NEXT:    tail __popcountsi2
 ;
 ; RV64I-LABEL: PR95271:
 ; RV64I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
index 197366e7e05fe..ffebc73a34690 100644
--- a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
@@ -11,6 +11,8 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    beqz a0, .LBB0_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 2
@@ -22,27 +24,9 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_2:
 ; RV32I-NEXT:    li a0, 32
@@ -61,14 +45,11 @@ declare i64 @llvm.ctlz.i64(i64, i1)
 define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctlz_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a4, a2, 1365
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a3, a2, 819
-; RV32I-NEXT:    lui a2, 61681
-; RV32I-NEXT:    addi a2, a2, -241
-; RV32I-NEXT:    bnez a1, .LBB1_2
-; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 2
@@ -80,27 +61,10 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    addi a0, a0, 32
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    srli a0, a1, 1
-; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    srli a0, s1, 1
+; RV32I-NEXT:    or a0, s1, a0
 ; RV32I-NEXT:    srli a1, a0, 2
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 4
@@ -110,22 +74,16 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    bnez s1, .LBB1_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    addi a0, s0, 32
+; RV32I-NEXT:  .LBB1_2:
 ; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32XTHEADBB-LABEL: ctlz_i64:
diff --git a/llvm/test/CodeGen/RISCV/rv32zbb.ll b/llvm/test/CodeGen/RISCV/rv32zbb.ll
index fa320f53cec6c..86e0d6b7b3f9d 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbb.ll
@@ -11,6 +11,8 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    beqz a0, .LBB0_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 2
@@ -22,27 +24,9 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_2:
 ; RV32I-NEXT:    li a0, 32
@@ -61,14 +45,11 @@ declare i64 @llvm.ctlz.i64(i64, i1)
 define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctlz_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a4, a2, 1365
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a3, a2, 819
-; RV32I-NEXT:    lui a2, 61681
-; RV32I-NEXT:    addi a2, a2, -241
-; RV32I-NEXT:    bnez a1, .LBB1_2
-; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 2
@@ -80,27 +61,10 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    addi a0, a0, 32
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    srli a0, a1, 1
-; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    srli a0, s1, 1
+; RV32I-NEXT:    or a0, s1, a0
 ; RV32I-NEXT:    srli a1, a0, 2
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 4
@@ -110,22 +74,16 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    and a1, a0, a3
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a3
-; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    bnez s1, .LBB1_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    addi a0, s0, 32
+; RV32I-NEXT:  .LBB1_2:
 ; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: ctlz_i64:
@@ -253,28 +211,7 @@ declare i32 @llvm.ctpop.i32(i32)
 define i32 @ctpop_i32(i32 %a) nounwind {
 ; RV32I-LABEL: ctpop_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a1, a0, 1
-; RV32I-NEXT:    lui a2, 349525
-; RV32I-NEXT:    addi a2, a2, 1365
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    sub a0, a0, a1
-; RV32I-NEXT:    lui a1, 209715
-; RV32I-NEXT:    addi a1, a1, 819
-; RV32I-NEXT:    and a2, a0, a1
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    add a0, a2, a0
-; RV32I-NEXT:    srli a1, a0, 4
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi a1, a1, -241
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    ret
+; RV32I-NEXT:    tail __popcountsi2
 ;
 ; RV32ZBB-LABEL: ctpop_i32:
 ; RV32ZBB:       # %bb.0:
@@ -365,42 +302,21 @@ declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
 define <2 x i32> @ctpop_v2i32(<2 x i32> %a) nounwind {
 ; RV32I-LABEL: ctpop_v2i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    lui a3, 349525
-; RV32I-NEXT:    addi a3, a3, 1365
-; RV32I-NEXT:    and a2, a2, a3
-; RV32I-NEXT:    sub a0, a0, a2
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a2, a2, 819
-; RV32I-NEXT:    and a4, a0, a2
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    add a0, a4, a0
-; RV32I-NEXT:    srli a4, a0, 4
-; RV32I-NEXT:    add a0, a0, a4
-; RV32I-NEXT:    lui a4, 61681
-; RV32I-NEXT:    addi a4, a4, -241
-; RV32I-NEXT:    and a0, a0, a4
-; RV32I-NEXT:    slli a5, a0, 8
-; RV32I-NEXT:    add a0, a0, a5
-; RV32I-NEXT:    slli a5, a0, 16
-; RV32I-NEXT:    add a0, a0, a5
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    srli a5, a1, 1
-; RV32I-NEXT:    and a3, a5, a3
-; RV32I-NEXT:    sub a1, a1, a3
-; RV32I-NEXT:    and a3, a1, a2
-; RV32I-NEXT:    srli a1, a1, 2
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    add a1, a3, a1
-; RV32I-NEXT:    srli a2, a1, 4
-; RV32I-NEXT:    add a1, a1, a2
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    slli a2, a1, 8
-; RV32I-NEXT:    add a1, a1, a2
-; RV32I-NEXT:    slli a2, a1, 16
-; RV32I-NEXT:    add a1, a1, a2
-; RV32I-NEXT:    srli a1, a1, 24
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    call __popcountsi2
+; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: ctpop_v2i32:
@@ -517,44 +433,11 @@ declare i64 @llvm.ctpop.i64(i64)
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctpop_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a2, a1, 1
-; RV32I-NEXT:    lui a3, 349525
-; RV32I-NEXT:    addi a3, a3, 1365
-; RV32I-NEXT:    and a2, a2, a3
-; RV32I-NEXT:    sub a1, a1, a2
-; RV32I-NEXT:    lui a2, 209715
-; RV32I-NEXT:    addi a2, a2, 819
-; RV32I-NEXT:    and a4, a1, a2
-; RV32I-NEXT:    srli a1, a1, 2
-; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    add a1, a4, a1
-; RV32I-NEXT:    srli a4, a1, 4
-; RV32I-NEXT:    add a1, a1, a4
-; RV32I-NEXT:    lui a4, 61681
-; RV32I-NEXT:    addi a4, a4, -241
-; RV32I-NEXT:    and a1, a1, a4
-; RV32I-NEXT:    slli a5, a1, 8
-; RV32I-NEXT:    add a1, a1, a5
-; RV32I-NEXT:    slli a5, a1, 16
-; RV32I-NEXT:    add a1, a1, a5
-; RV32I-NEXT:    srli a1, a1, 24
-; RV32I-NEXT:    srli a5, a0, 1
-; RV32I-NEXT:    and a3, a5, a3
-; RV32I-NEXT:    sub a0, a0, a3
-; RV32I-NEXT:    and a3, a0, a2
-; RV32I-NEXT:    srli a0, a0, 2
-; RV32I-NEXT:    and a0, a0, a2
-; RV32I-NEXT:    add a0, a3, a0
-; RV32I-NEXT:    srli a2, a0, 4
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    and a0, a0, a4
-; RV32I-NEXT:    slli a2, a0, 8
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    slli a2, a0, 16
-; RV32I-NEXT:    add a0, a0, a2
-; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    li a1, 0
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __popcountdi2
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: ctpop_i64:
@@ -682,82 +565,36 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
 define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV32I-LABEL: ctpop_v2i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lw a3, 4(a1)
-; RV32I-NEXT:    lw a2, 8(a1)
-; RV32I-NEXT:    lw a4, 12(a1)
-; RV32I-NEXT:    lw a1, 0(a1)
-; RV32I-NEXT:    srli a5, a3, 1
-; RV32I-NEXT:    lui a6, 349525
-; RV32I-NEXT:    addi a6, a6, 1365
-; RV32I-NEXT:    and a5, a5, a6
-; RV32I-NEXT:    sub a3, a3, a5
-; RV32I-NEXT:    lui a5, 209715
-; RV32I-NEXT:    addi a5, a5, 819
-; RV32I-NEXT:    and a7, a3, a5
-; RV32I-NEXT:    srli a3, a3, 2
-; RV32I-NEXT:    and a3, a3, a5
-; RV32I-NEXT:    add a3, a7, a3
-; RV32I-NEXT:    srli a7, a3, 4
-; RV32I-NEXT:    add a3, a3, a7
-; RV32I-NEXT:    lui a7, 61681
-; RV32I-NEXT:    addi a7, a7, -241
-; RV32I-NEXT:    and a3, a3, a7
-; RV32I-NEXT:    slli t0, a3, 8
-; RV32I-NEXT:    add a3, a3, t0
-; RV32I-NEXT:    slli t0, a3, 16
-; RV32I-NEXT:    add a3, a3, t0
-; RV32I-NEXT:    srli a3, a3, 24
-; RV32I-NEXT:    srli t0, a1, 1
-; RV32I-NEXT:    and t0, t0, a6
-; RV32I-NEXT:    sub a1, a1, t0
-; RV32I-NEXT:    and t0, a1, a5
-; RV32I-NEXT:    srli a1, a1, 2
-; RV32I-NEXT:    and a1, a1, a5
-; RV32I-NEXT:    add a1, t0, a1
-; RV32I-NEXT:    srli t0, a1, 4
-; RV32I-NEXT:    add a1, a1, t0
-; RV32I-NEXT:    and a1, a1, a7
-; RV32I-NEXT:    slli t0, a1, 8
-; RV32I-NEXT:    add a1, a1, t0
-; RV32I-NEXT:    slli t0, a1, 16
-; RV32I-NEXT:    add a1, a1, t0
-; RV32I-NEXT:    srli a1, a1, 24
-; RV32I-NEXT:    add a1, a1, a3
-; RV32I-NEXT:    srli a3, a4, 1
-; RV32I-NEXT:    and a3, a3, a6
-; RV32I-NEXT:    sub a4, a4, a3
-; RV32I-NEXT:    and a3, a4, a5
-; RV32I-NEXT:    srli a4, a4, 2
-; RV32I-NEXT:    and a4, a4, a5
-; RV32I-NEXT:    add a3, a3, a4
-; RV32I-NEXT:    srli a4, a3, 4
-; RV32I-NEXT:    add a3, a3, a4
-; RV32I-NEXT:    and a3, a3, a7
-; RV32I-NEXT:    slli a4, a3, 8
-; RV32I-NEXT:    add a3, a3, a4
-; RV32I-NEXT:    slli a4, a3, 16
-; RV32I-NEXT:    add a3, a3, a4
-; RV32I-NEXT:    srli a3, a3, 24
-; RV32I-NEXT:    srli a4, a2, 1
-; RV32I-NEXT:    and a4, a4, a6
-; RV32I-NEXT:    sub a2, a2, a4
-; RV32I-NEXT:    and a4, a2, a5
-; RV32I-NEXT:    srli a2, a2, 2
-; RV32I-NEXT:    and a2, a2, a5
-; RV32I-NEXT:    add a2, a4, a2
-; RV32I-NEXT:    srli a4, a2, 4
-; RV32I-NEXT:    add a2, a2, a4
-; RV32I-NEXT:    and a2, a2, a7
-; RV32I-NEXT:    slli a4, a2, 8
-; RV32I-NEXT:    add a2, a2, a4
-; RV32I-NEXT:    slli a4, a2, 16
-; RV32I-NEXT:    add a2, a2, a4
-; RV32I-NEXT:    srli a2, a2, 24
-; RV32I-NEXT:    add a2, a2, a3
-; RV32I-NEXT:    sw zero, 12(a0)
-; RV32I-NEXT:    sw zero, 4(a0)
-; RV32I-NEXT:    sw a2, 8(a0)
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    lw s0, 8(a1)
+; RV32I-NEXT:    lw s1, 12(a1)
+; RV32I-NEXT:    lw a2, 0(a1)
+; RV32I-NEXT:    lw a1, 4(a1)
+; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    call __popcountdi2
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    call __popcountdi2
+; RV32I-NEXT:    sw a1, 12(s2)
+; RV32I-NEXT:    sw a0, 8(s2)
+; RV32I-NEXT:    sw s4, 4(s2)
+; RV32I-NEXT:    sw s3, 0(s2)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: ctpop_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
index 80d3add385969..eb8b5c9dd695f 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
@@ -317,6 +317,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    beqz a0, .LBB5_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -330,35 +332,9 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB5_2:
 ; RV64I-NEXT:    li a0, 64
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
index b0e447b71178b..445b4dc671296 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
@@ -307,6 +307,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    beqz a0, .LBB5_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -320,35 +322,9 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB5_2:
 ; RV64I-NEXT:    li a0, 64
@@ -620,36 +596,7 @@ declare i64 @llvm.ctpop.i64(i64)
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ret
+; RV64I-NEXT:    tail __popcountdi2
 ;
 ; RV64ZBB-LABEL: ctpop_i64:
 ; RV64ZBB:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 6cdab888ffcde..c667b930c1ee0 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -295,6 +295,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    beqz a0, .LBB5_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -308,35 +310,9 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB5_2:
 ; RV64I-NEXT:    li a0, 64
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 43a499806ab5a..53d7f77285e72 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -285,6 +285,8 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    beqz a0, .LBB5_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    srli a1, a0, 2
@@ -298,35 +300,9 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a1, a0, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    not a0, a0
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB5_2:
 ; RV64I-NEXT:    li a0, 64
@@ -828,36 +804,7 @@ declare i64 @llvm.ctpop.i64(i64)
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    lui a2, 349525
-; RV64I-NEXT:    addiw a2, a2, 1365
-; RV64I-NEXT:    slli a3, a2, 32
-; RV64I-NEXT:    add a2, a2, a3
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw a1, a1, 819
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a2, a0, a1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    add a0, a2, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw a1, a1, -241
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    slli a1, a0, 32
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ret
+; RV64I-NEXT:    tail __popcountdi2
 ;
 ; RV64ZBB-LABEL: ctpop_i64:
 ; RV64ZBB:       # %bb.0:
@@ -948,52 +895,21 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
 define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV64I-LABEL: ctpop_v2i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srli a2, a0, 1
-; RV64I-NEXT:    lui a3, 349525
-; RV64I-NEXT:    addiw a3, a3, 1365
-; RV64I-NEXT:    slli a4, a3, 32
-; RV64I-NEXT:    add a3, a3, a4
-; RV64I-NEXT:    and a2, a2, a3
-; RV64I-NEXT:    sub a0, a0, a2
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    addiw a2, a2, 819
-; RV64I-NEXT:    slli a4, a2, 32
-; RV64I-NEXT:    add a2, a2, a4
-; RV64I-NEXT:    and a4, a0, a2
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, a2
-; RV64I-NEXT:    add a0, a4, a0
-; RV64I-NEXT:    srli a4, a0, 4
-; RV64I-NEXT:    add a0, a0, a4
-; RV64I-NEXT:    lui a4, 61681
-; RV64I-NEXT:    addiw a4, a4, -241
-; RV64I-NEXT:    slli a5, a4, 32
-; RV64I-NEXT:    add a4, a4, a5
-; RV64I-NEXT:    and a0, a0, a4
-; RV64I-NEXT:    slli a5, a0, 8
-; RV64I-NEXT:    add a0, a0, a5
-; RV64I-NEXT:    slli a5, a0, 16
-; RV64I-NEXT:    add a0, a0, a5
-; RV64I-NEXT:    slli a5, a0, 32
-; RV64I-NEXT:    add a0, a0, a5
-; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    srli a5, a1, 1
-; RV64I-NEXT:    and a3, a5, a3
-; RV64I-NEXT:    sub a1, a1, a3
-; RV64I-NEXT:    and a3, a1, a2
-; RV64I-NEXT:    srli a1, a1, 2
-; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    add a1, a3, a1
-; RV64I-NEXT:    srli a2, a1, 4
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    and a1, a1, a4
-; RV64I-NEXT:    slli a2, a1, 8
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    slli a2, a1, 16
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    slli a2, a1, 32
-; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    srli a1, a1, 56
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: ctpop_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/sextw-removal.ll b/llvm/test/CodeGen/RISCV/sextw-removal.ll
index 8cf78551d28f9..f9a2c543d5410 100644
--- a/llvm/test/CodeGen/RISCV/sextw-removal.ll
+++ b/llvm/test/CodeGen/RISCV/sextw-removal.ll
@@ -316,52 +316,18 @@ declare float @baz(i32 signext %i3)
 define void @test7(i32 signext %arg, i32 signext %arg1) nounwind {
 ; RV64I-LABEL: test7:
 ; RV64I:       # %bb.0: # %bb
-; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sraw a0, a0, a1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    addiw s0, a1, 1365
-; RV64I-NEXT:    slli a1, s0, 32
-; RV64I-NEXT:    add s0, s0, a1
-; RV64I-NEXT:    lui a1, 209715
-; RV64I-NEXT:    addiw s1, a1, 819
-; RV64I-NEXT:    slli a1, s1, 32
-; RV64I-NEXT:    add s1, s1, a1
-; RV64I-NEXT:    lui a1, 61681
-; RV64I-NEXT:    addiw s2, a1, -241
-; RV64I-NEXT:    slli a1, s2, 32
-; RV64I-NEXT:    add s2, s2, a1
-; RV64I-NEXT:    lui a1, 4112
-; RV64I-NEXT:    addiw s3, a1, 257
-; RV64I-NEXT:    slli a1, s3, 32
-; RV64I-NEXT:    add s3, s3, a1
 ; RV64I-NEXT:  .LBB6_1: # %bb2
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    call foo
-; RV64I-NEXT:    srli a1, a0, 1
-; RV64I-NEXT:    and a1, a1, s0
-; RV64I-NEXT:    sub a0, a0, a1
-; RV64I-NEXT:    and a1, a0, s1
-; RV64I-NEXT:    srli a0, a0, 2
-; RV64I-NEXT:    and a0, a0, s1
-; RV64I-NEXT:    add a0, a1, a0
-; RV64I-NEXT:    srli a1, a0, 4
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    and a0, a0, s2
-; RV64I-NEXT:    mul a0, a0, s3
-; RV64I-NEXT:    srli a0, a0, 56
+; RV64I-NEXT:    call __popcountdi2
 ; RV64I-NEXT:    bnez a0, .LBB6_1
 ; RV64I-NEXT:  # %bb.2: # %bb7
-; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: test7:
diff --git a/llvm/test/CodeGen/Thumb2/mve-ctpop.ll b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
index 724bd4f7963b8..670568f50a6fe 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; NOTE: Assertions have been autoenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
 
 define arm_aapcs_vfpcc <2 x i64> @ctpop_2i64_t(<2 x i64> %src){
@@ -7,56 +6,19 @@ define arm_aapcs_vfpcc <2 x i64> @ctpop_2i64_t(<2 x i64> %src){
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
-; CHECK-NEXT:    vmov r1, r2, d1
-; CHECK-NEXT:    mov.w lr, #1431655765
-; CHECK-NEXT:    vmov r3, r4, d0
-; CHECK-NEXT:    mov.w r12, #858993459
-; CHECK-NEXT:    vldr s1, .LCPI0_0
-; CHECK-NEXT:    vmov.f32 s3, s1
-; CHECK-NEXT:    and.w r0, lr, r2, lsr #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    and.w r2, r12, r0, lsr #2
-; CHECK-NEXT:    bic r0, r0, #-858993460
-; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    and.w r2, lr, r1, lsr #1
-; CHECK-NEXT:    subs r1, r1, r2
-; CHECK-NEXT:    add.w r0, r0, r0, lsr #4
-; CHECK-NEXT:    and.w r2, r12, r1, lsr #2
-; CHECK-NEXT:    bic r1, r1, #-858993460
-; CHECK-NEXT:    add r1, r2
-; CHECK-NEXT:    and.w r2, lr, r3, lsr #1
-; CHECK-NEXT:    subs r2, r3, r2
-; CHECK-NEXT:    bic r5, r0, #-252645136
-; CHECK-NEXT:    add.w r1, r1, r1, lsr #4
-; CHECK-NEXT:    mov.w r0, #16843009
-; CHECK-NEXT:    and.w r3, r12, r2, lsr #2
-; CHECK-NEXT:    bic r2, r2, #-858993460
-; CHECK-NEXT:    add r2, r3
-; CHECK-NEXT:    and.w r3, lr, r4, lsr #1
-; CHECK-NEXT:    subs r3, r4, r3
-; CHECK-NEXT:    bic r1, r1, #-252645136
-; CHECK-NEXT:    add.w r2, r2, r2, lsr #4
-; CHECK-NEXT:    muls r5, r0, r5
-; CHECK-NEXT:    and.w r4, r12, r3, lsr #2
-; CHECK-NEXT:    bic r3, r3, #-858993460
-; CHECK-NEXT:    bic r2, r2, #-252645136
-; CHECK-NEXT:    add r3, r4
-; CHECK-NEXT:    muls r1, r0, r1
-; CHECK-NEXT:    add.w r3, r3, r3, lsr #4
-; CHECK-NEXT:    muls r2, r0, r2
-; CHECK-NEXT:    bic r3, r3, #-252645136
-; CHECK-NEXT:    muls r0, r3, r0
-; CHECK-NEXT:    lsrs r1, r1, #24
-; CHECK-NEXT:    add.w r1, r1, r5, lsr #24
-; CHECK-NEXT:    lsrs r2, r2, #24
-; CHECK-NEXT:    vmov s2, r1
-; CHECK-NEXT:    add.w r0, r2, r0, lsr #24
-; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, r1, d9
+; CHECK-NEXT:    bl __popcountdi2
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    bl __popcountdi2
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
-; CHECK-NEXT:    .p2align 2
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI0_0:
-; CHECK-NEXT:    .long 0x00000000 @ float 0
 entry:
   %0 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %src)
   ret <2 x i64> %0

>From ae98e9698e4e91e7b7c117fc62b3b29057d9c4af Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88 at gmail.com>
Date: Sat, 3 Aug 2024 06:46:40 +0300
Subject: [PATCH 2/3] Tests

---
 llvm/test/CodeGen/ARM/popcnt.ll               |   5 +-
 llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll    | 149 ++++++++++++++++++
 .../CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll   |  66 ++++++--
 llvm/test/CodeGen/RISCV/rv64zbb.ll            |  66 ++++++--
 4 files changed, 251 insertions(+), 35 deletions(-)

diff --git a/llvm/test/CodeGen/ARM/popcnt.ll b/llvm/test/CodeGen/ARM/popcnt.ll
index 6c01c516be9d2..8178b9e25f5c5 100644
--- a/llvm/test/CodeGen/ARM/popcnt.ll
+++ b/llvm/test/CodeGen/ARM/popcnt.ll
@@ -329,7 +329,7 @@ define i32 @ctpop32(i32 %x) nounwind readnone {
   ret i32 %count
 }
 
-define i32 @ctpop64(i64 %x) nounwind readnone {
+define i64 @ctpop64(i64 %x) nounwind readnone {
 ; CHECK-LABEL: ctpop64:
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r11, lr}
@@ -338,8 +338,7 @@ define i32 @ctpop64(i64 %x) nounwind readnone {
 ; CHECK-NEXT:    pop {r11, lr}
 ; CHECK-NEXT:    mov pc, lr
   %count = tail call i64 @llvm.ctpop.i64(i64 %x)
-  %conv = trunc i64 %count to i32
-  ret i32 %conv
+  ret i64 %count
 }
 
 define i32 @ctpop_eq_one(i64 %x) nounwind readnone {
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 9123017918094..d5d167758fb34 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -28,6 +28,7 @@ declare i8 @llvm.ctpop.i8(i8)
 declare i16 @llvm.ctpop.i16(i16)
 declare i32 @llvm.ctpop.i32(i32)
 declare i64 @llvm.ctpop.i64(i64)
+declare i128 @llvm.ctpop.i128(i128)
 
 define i8 @test_cttz_i8(i8 %a) nounwind {
 ; RV32_NOZBB-LABEL: test_cttz_i8:
@@ -2094,6 +2095,154 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
   ret i64 %1
 }
 
+define i128 @test_ctpop_i128(i128 %a) nounwind {
+; RV32_NOZBB-LABEL: test_ctpop_i128:
+; RV32_NOZBB:       # %bb.0:
+; RV32_NOZBB-NEXT:    addi sp, sp, -32
+; RV32_NOZBB-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32_NOZBB-NEXT:    lw s0, 0(a1)
+; RV32_NOZBB-NEXT:    lw s1, 4(a1)
+; RV32_NOZBB-NEXT:    lw a2, 8(a1)
+; RV32_NOZBB-NEXT:    lw a1, 12(a1)
+; RV32_NOZBB-NEXT:    mv s2, a0
+; RV32_NOZBB-NEXT:    mv a0, a2
+; RV32_NOZBB-NEXT:    call __popcountdi2
+; RV32_NOZBB-NEXT:    mv s3, a0
+; RV32_NOZBB-NEXT:    mv s4, a1
+; RV32_NOZBB-NEXT:    mv a0, s0
+; RV32_NOZBB-NEXT:    mv a1, s1
+; RV32_NOZBB-NEXT:    call __popcountdi2
+; RV32_NOZBB-NEXT:    add a1, a1, s4
+; RV32_NOZBB-NEXT:    add s3, a0, s3
+; RV32_NOZBB-NEXT:    sltu a0, s3, a0
+; RV32_NOZBB-NEXT:    add a0, a1, a0
+; RV32_NOZBB-NEXT:    sw zero, 12(s2)
+; RV32_NOZBB-NEXT:    sw zero, 8(s2)
+; RV32_NOZBB-NEXT:    sw s3, 0(s2)
+; RV32_NOZBB-NEXT:    sw a0, 4(s2)
+; RV32_NOZBB-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32_NOZBB-NEXT:    addi sp, sp, 32
+; RV32_NOZBB-NEXT:    ret
+;
+; RV64NOZBB-LABEL: test_ctpop_i128:
+; RV64NOZBB:       # %bb.0:
+; RV64NOZBB-NEXT:    addi sp, sp, -32
+; RV64NOZBB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    mv s0, a0
+; RV64NOZBB-NEXT:    mv a0, a1
+; RV64NOZBB-NEXT:    call __popcountdi2
+; RV64NOZBB-NEXT:    mv s1, a0
+; RV64NOZBB-NEXT:    mv a0, s0
+; RV64NOZBB-NEXT:    call __popcountdi2
+; RV64NOZBB-NEXT:    add a0, a0, s1
+; RV64NOZBB-NEXT:    li a1, 0
+; RV64NOZBB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    addi sp, sp, 32
+; RV64NOZBB-NEXT:    ret
+;
+; RV32ZBB-LABEL: test_ctpop_i128:
+; RV32ZBB:       # %bb.0:
+; RV32ZBB-NEXT:    lw a2, 12(a1)
+; RV32ZBB-NEXT:    lw a3, 8(a1)
+; RV32ZBB-NEXT:    lw a4, 0(a1)
+; RV32ZBB-NEXT:    lw a1, 4(a1)
+; RV32ZBB-NEXT:    cpop a2, a2
+; RV32ZBB-NEXT:    cpop a3, a3
+; RV32ZBB-NEXT:    add a2, a3, a2
+; RV32ZBB-NEXT:    cpop a1, a1
+; RV32ZBB-NEXT:    cpop a3, a4
+; RV32ZBB-NEXT:    add a1, a3, a1
+; RV32ZBB-NEXT:    add a2, a1, a2
+; RV32ZBB-NEXT:    sltu a1, a2, a1
+; RV32ZBB-NEXT:    sw zero, 12(a0)
+; RV32ZBB-NEXT:    sw zero, 8(a0)
+; RV32ZBB-NEXT:    sw a2, 0(a0)
+; RV32ZBB-NEXT:    sw a1, 4(a0)
+; RV32ZBB-NEXT:    ret
+;
+; RV64ZBB-LABEL: test_ctpop_i128:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    li a1, 0
+; RV64ZBB-NEXT:    ret
+;
+; RV32XTHEADBB-LABEL: test_ctpop_i128:
+; RV32XTHEADBB:       # %bb.0:
+; RV32XTHEADBB-NEXT:    addi sp, sp, -32
+; RV32XTHEADBB-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32XTHEADBB-NEXT:    lw s0, 0(a1)
+; RV32XTHEADBB-NEXT:    lw s1, 4(a1)
+; RV32XTHEADBB-NEXT:    lw a2, 8(a1)
+; RV32XTHEADBB-NEXT:    lw a1, 12(a1)
+; RV32XTHEADBB-NEXT:    mv s2, a0
+; RV32XTHEADBB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NEXT:    call __popcountdi2
+; RV32XTHEADBB-NEXT:    mv s3, a0
+; RV32XTHEADBB-NEXT:    mv s4, a1
+; RV32XTHEADBB-NEXT:    mv a0, s0
+; RV32XTHEADBB-NEXT:    mv a1, s1
+; RV32XTHEADBB-NEXT:    call __popcountdi2
+; RV32XTHEADBB-NEXT:    add a1, a1, s4
+; RV32XTHEADBB-NEXT:    add s3, a0, s3
+; RV32XTHEADBB-NEXT:    sltu a0, s3, a0
+; RV32XTHEADBB-NEXT:    add a0, a1, a0
+; RV32XTHEADBB-NEXT:    sw zero, 12(s2)
+; RV32XTHEADBB-NEXT:    sw zero, 8(s2)
+; RV32XTHEADBB-NEXT:    sw s3, 0(s2)
+; RV32XTHEADBB-NEXT:    sw a0, 4(s2)
+; RV32XTHEADBB-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32XTHEADBB-NEXT:    addi sp, sp, 32
+; RV32XTHEADBB-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: test_ctpop_i128:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    addi sp, sp, -32
+; RV64XTHEADBB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64XTHEADBB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64XTHEADBB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64XTHEADBB-NEXT:    mv s0, a0
+; RV64XTHEADBB-NEXT:    mv a0, a1
+; RV64XTHEADBB-NEXT:    call __popcountdi2
+; RV64XTHEADBB-NEXT:    mv s1, a0
+; RV64XTHEADBB-NEXT:    mv a0, s0
+; RV64XTHEADBB-NEXT:    call __popcountdi2
+; RV64XTHEADBB-NEXT:    add a0, a0, s1
+; RV64XTHEADBB-NEXT:    li a1, 0
+; RV64XTHEADBB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64XTHEADBB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64XTHEADBB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64XTHEADBB-NEXT:    addi sp, sp, 32
+; RV64XTHEADBB-NEXT:    ret
+  %1 = call i128 @llvm.ctpop.i128(i128 %a)
+  ret i128 %1
+}
+
 define i8 @test_parity_i8(i8 %a) {
 ; RV32_NOZBB-LABEL: test_parity_i8:
 ; RV32_NOZBB:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
index 445b4dc671296..ca04cdd8c8d09 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
@@ -606,6 +606,40 @@ define i64 @ctpop_i64(i64 %a) nounwind {
   ret i64 %1
 }
 
+declare i128 @llvm.ctpop.i128(i128)
+
+define i128 @ctpop_i128(i128 %a) nounwind {
+; RV64I-LABEL: ctpop_i128:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    add a0, a0, s1
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i128:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    li a1, 0
+; RV64ZBB-NEXT:    ret
+  %1 = call i128 @llvm.ctpop.i128(i128 %a)
+  ret i128 %1
+}
+
 define signext i32 @sextb_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: sextb_i32:
 ; RV64I:       # %bb.0:
@@ -673,10 +707,10 @@ define i64 @sexth_i64(i64 %a) nounwind {
 define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: min_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB18_2
+; RV64I-NEXT:    blt a0, a1, .LBB19_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB18_2:
+; RV64I-NEXT:  .LBB19_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i32:
@@ -691,10 +725,10 @@ define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @min_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: min_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB19_2
+; RV64I-NEXT:    blt a0, a1, .LBB20_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB19_2:
+; RV64I-NEXT:  .LBB20_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i64:
@@ -709,10 +743,10 @@ define i64 @min_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: max_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB20_2
+; RV64I-NEXT:    blt a1, a0, .LBB21_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB20_2:
+; RV64I-NEXT:  .LBB21_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i32:
@@ -727,10 +761,10 @@ define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @max_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: max_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB21_2
+; RV64I-NEXT:    blt a1, a0, .LBB22_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB21_2:
+; RV64I-NEXT:  .LBB22_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i64:
@@ -745,10 +779,10 @@ define i64 @max_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: minu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB22_2
+; RV64I-NEXT:    bltu a0, a1, .LBB23_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB22_2:
+; RV64I-NEXT:  .LBB23_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i32:
@@ -763,10 +797,10 @@ define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: minu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB23_2
+; RV64I-NEXT:    bltu a0, a1, .LBB24_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB23_2:
+; RV64I-NEXT:  .LBB24_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i64:
@@ -781,10 +815,10 @@ define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: maxu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB24_2
+; RV64I-NEXT:    bltu a1, a0, .LBB25_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB24_2:
+; RV64I-NEXT:  .LBB25_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i32:
@@ -799,10 +833,10 @@ define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @maxu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: maxu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB25_2
+; RV64I-NEXT:    bltu a1, a0, .LBB26_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB25_2:
+; RV64I-NEXT:  .LBB26_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i64:
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 53d7f77285e72..0ced55b0dd63c 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -890,6 +890,40 @@ define i1 @ctpop_i64_ne_one(i64 %a) nounwind {
   ret i1 %2
 }
 
+declare i128 @llvm.ctpop.i128(i128)
+
+define i128 @ctpop_i128(i128 %a) nounwind {
+; RV64I-LABEL: ctpop_i128:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    add a0, a0, s1
+; RV64I-NEXT:    li a1, 0
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+;
+; RV64ZBB-LABEL: ctpop_i128:
+; RV64ZBB:       # %bb.0:
+; RV64ZBB-NEXT:    cpop a1, a1
+; RV64ZBB-NEXT:    cpop a0, a0
+; RV64ZBB-NEXT:    add a0, a0, a1
+; RV64ZBB-NEXT:    li a1, 0
+; RV64ZBB-NEXT:    ret
+  %1 = call i128 @llvm.ctpop.i128(i128 %a)
+  ret i128 %1
+}
+
 declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
 
 define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
@@ -1088,10 +1122,10 @@ define i64 @sexth_i64(i64 %a) nounwind {
 define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: min_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB36_2
+; RV64I-NEXT:    blt a0, a1, .LBB37_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB36_2:
+; RV64I-NEXT:  .LBB37_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i32:
@@ -1106,10 +1140,10 @@ define signext i32 @min_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @min_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: min_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a0, a1, .LBB37_2
+; RV64I-NEXT:    blt a0, a1, .LBB38_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB37_2:
+; RV64I-NEXT:  .LBB38_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: min_i64:
@@ -1124,10 +1158,10 @@ define i64 @min_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: max_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB38_2
+; RV64I-NEXT:    blt a1, a0, .LBB39_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB38_2:
+; RV64I-NEXT:  .LBB39_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i32:
@@ -1142,10 +1176,10 @@ define signext i32 @max_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @max_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: max_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    blt a1, a0, .LBB39_2
+; RV64I-NEXT:    blt a1, a0, .LBB40_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB39_2:
+; RV64I-NEXT:  .LBB40_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: max_i64:
@@ -1160,10 +1194,10 @@ define i64 @max_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: minu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB40_2
+; RV64I-NEXT:    bltu a0, a1, .LBB41_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB40_2:
+; RV64I-NEXT:  .LBB41_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i32:
@@ -1178,10 +1212,10 @@ define signext i32 @minu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: minu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a0, a1, .LBB41_2
+; RV64I-NEXT:    bltu a0, a1, .LBB42_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB41_2:
+; RV64I-NEXT:  .LBB42_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: minu_i64:
@@ -1196,10 +1230,10 @@ define i64 @minu_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: maxu_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB42_2
+; RV64I-NEXT:    bltu a1, a0, .LBB43_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB42_2:
+; RV64I-NEXT:  .LBB43_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i32:
@@ -1214,10 +1248,10 @@ define signext i32 @maxu_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @maxu_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: maxu_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    bltu a1, a0, .LBB43_2
+; RV64I-NEXT:    bltu a1, a0, .LBB44_2
 ; RV64I-NEXT:  # %bb.1:
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:  .LBB43_2:
+; RV64I-NEXT:  .LBB44_2:
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: maxu_i64:

>From 14340fe58feb8e182985ef9f4bdd8e04c3c3690e Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88 at gmail.com>
Date: Sat, 3 Aug 2024 06:47:58 +0300
Subject: [PATCH 3/3] Fixes

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 93 +++++++++++++-----
 .../SelectionDAG/LegalizeIntegerTypes.cpp     |  6 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  4 +-
 llvm/test/CodeGen/ARM/popcnt.ll               |  1 +
 llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll    | 97 +++++++++----------
 llvm/test/CodeGen/RISCV/rv32zbb.ll            |  8 +-
 .../CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll   | 29 +++---
 llvm/test/CodeGen/RISCV/rv64zbb.ll            | 29 +++---
 llvm/test/CodeGen/Thumb2/mve-ctpop.ll         |  3 +-
 9 files changed, 155 insertions(+), 115 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index c91a2360d1599..8a10f95ca20f6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -129,7 +129,8 @@ class SelectionDAGLegalize {
                                      ArrayRef<int> Mask) const;
 
   std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
-                        TargetLowering::ArgListTy &&Args, bool isSigned);
+                                            TargetLowering::ArgListTy &&Args,
+                                            bool IsSigned, EVT RetVT);
   std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
 
   void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
@@ -140,14 +141,20 @@ class SelectionDAGLegalize {
                        RTLIB::Libcall Call_F128,
                        RTLIB::Libcall Call_PPCF128,
                        SmallVectorImpl<SDValue> &Results);
-  SDValue ExpandIntLibCall(SDNode *Node, bool IsSigned, RTLIB::Libcall Call_I8,
-                           RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
-                           RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
+  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
+                           RTLIB::Libcall Call_I8,
+                           RTLIB::Libcall Call_I16,
+                           RTLIB::Libcall Call_I32,
+                           RTLIB::Libcall Call_I64,
+                           RTLIB::Libcall Call_I128);
   void ExpandArgFPLibCall(SDNode *Node,
                           RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
                           RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
                           RTLIB::Libcall Call_PPCF128,
                           SmallVectorImpl<SDValue> &Results);
+  SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
+                                   RTLIB::Libcall CallI64,
+                                   RTLIB::Libcall CallI128);
   void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
   void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
 
@@ -2056,9 +2063,10 @@ SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
 // register, return the lo part and set the hi part to the by-reg argument in
 // the first.  If it does fit into a single register, return the result and
 // leave the Hi part unset.
-std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
-                                            TargetLowering::ArgListTy &&Args,
-                                            bool isSigned) {
+std::pair<SDValue, SDValue>
+SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
+                                    TargetLowering::ArgListTy &&Args,
+                                    bool IsSigned, EVT RetVT) {
   EVT CodePtrTy = TLI.getPointerTy(DAG.getDataLayout());
   SDValue Callee;
   if (const char *LibcallName = TLI.getLibcallName(LC))
@@ -2069,7 +2077,6 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall L
                                 Node->getOperationName(&DAG));
   }
 
-  EVT RetVT = Node->getValueType(0);
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
 
   // By default, the input chain to this libcall is the entry node of the
@@ -2089,7 +2096,7 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall L
     InChain = TCChain;
 
   TargetLowering::CallLoweringInfo CLI(DAG);
-  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
+  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, IsSigned);
   CLI.setDebugLoc(SDLoc(Node))
       .setChain(InChain)
       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
@@ -2125,7 +2132,8 @@ std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall L
     Args.push_back(Entry);
   }
 
-  return ExpandLibCall(LC, Node, std::move(Args), isSigned);
+  return ExpandLibCall(LC, Node, std::move(Args), isSigned,
+                       Node->getValueType(0));
 }
 
 void SelectionDAGLegalize::ExpandFrexpLibCall(
@@ -2152,7 +2160,8 @@ void SelectionDAGLegalize::ExpandFrexpLibCall(
   TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
 
   RTLIB::Libcall LC = RTLIB::getFREXP(VT);
-  auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
+  auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args),
+                                     /*IsSigned=*/false, VT);
 
   // FIXME: Get type of int for libcall declaration and cast
 
@@ -2206,7 +2215,7 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
   ExpandFPLibCall(Node, LC, Results);
 }
 
-SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode *Node, bool IsSigned,
+SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
                                                RTLIB::Libcall Call_I8,
                                                RTLIB::Libcall Call_I16,
                                                RTLIB::Libcall Call_I32,
@@ -2221,9 +2230,7 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode *Node, bool IsSigned,
   case MVT::i64:  LC = Call_I64; break;
   case MVT::i128: LC = Call_I128; break;
   }
-  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
-         "LibCall explicitly requested, but not available");
-  return ExpandLibCall(LC, Node, IsSigned).first;
+  return ExpandLibCall(LC, Node, isSigned).first;
 }
 
 /// Expand the node to a libcall based on first argument type (for instance
@@ -2242,6 +2249,50 @@ void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
   ExpandFPLibCall(Node, LC, Results);
 }
 
+SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
+    SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
+    RTLIB::Libcall CallI128) {
+  RTLIB::Libcall LC;
+  switch (Node->getSimpleValueType(0).SimpleTy) {
+  default:
+    llvm_unreachable("Unexpected request for libcall!");
+  case MVT::i32:
+    LC = CallI32;
+    break;
+  case MVT::i64:
+    LC = CallI64;
+    break;
+  case MVT::i128:
+    LC = CallI128;
+    break;
+  }
+
+  // Bit-counting libcalls have one unsigned argument and return `int`.
+  // Note that `int` may be illegal on this target; ExpandLibCall will
+  // take care of promoting it to a legal type.
+  SDValue Op = Node->getOperand(0);
+  EVT IntVT =
+      EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
+
+  TargetLowering::ArgListEntry Arg;
+  EVT ArgVT = Op.getValueType();
+  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
+  Arg.Node = Op;
+  Arg.Ty = ArgTy;
+  Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, /*IsSigned=*/false);
+  Arg.IsZExt = !Arg.IsSExt;
+
+  SDValue Res = ExpandLibCall(LC, Node, TargetLowering::ArgListTy{Arg},
+                              /*IsSigned=*/true, IntVT)
+                    .first;
+
+  // If ExpandLibCall created a tail call, the result was already
+  // of the correct type. Otherwise, we need to sign extend it.
+  if (Res.getValueType() != MVT::Other)
+    Res = DAG.getSExtOrTrunc(Res, SDLoc(Node), Node->getValueType(0));
+  return Res;
+}
+
 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
 void
 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
@@ -4999,16 +5050,12 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
     break;
   case ISD::CTLZ_ZERO_UNDEF:
-    Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
-                                       RTLIB::UNKNOWN_LIBCALL,
-                                       RTLIB::UNKNOWN_LIBCALL, RTLIB::CTLZ_I32,
-                                       RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
+    Results.push_back(ExpandBitCountingLibCall(
+        Node, RTLIB::CTLZ_I32, RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
     break;
   case ISD::CTPOP:
-    Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
-                                       RTLIB::UNKNOWN_LIBCALL,
-                                       RTLIB::UNKNOWN_LIBCALL, RTLIB::CTPOP_I32,
-                                       RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
+    Results.push_back(ExpandBitCountingLibCall(
+        Node, RTLIB::CTPOP_I32, RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
     break;
   case ISD::RESET_FPENV: {
     // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index d00785025bac5..0b0789b75dd50 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3866,8 +3866,10 @@ void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
     assert(LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC) &&
            "LibCall explicitly requested, but not available");
     TargetLowering::MakeLibCallOptions CallOptions;
-    SDValue Res = TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, DL).first;
-    SplitInteger(Res, Lo, Hi);
+    EVT IntVT =
+        EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
+    SDValue Res = TLI.makeLibCall(DAG, LC, IntVT, Op, CallOptions, DL).first;
+    SplitInteger(DAG.getSExtOrTrunc(Res, DL, VT), Lo, Hi);
     return;
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4a2193c8d5328..c16b1ca742d7b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -394,7 +394,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     }
   } else {
     setOperationAction(ISD::CTTZ, XLenVT, Expand);
-    if (!Subtarget.is64Bit())
+    if (Subtarget.is64Bit())
+      setOperationAction(ISD::CTPOP, MVT::i128, LibCall);
+    else
       setOperationAction(ISD::CTPOP, MVT::i32, LibCall);
     setOperationAction(ISD::CTPOP, MVT::i64, LibCall);
     if (RV64LegalI32 && Subtarget.is64Bit())
diff --git a/llvm/test/CodeGen/ARM/popcnt.ll b/llvm/test/CodeGen/ARM/popcnt.ll
index 8178b9e25f5c5..fc4387320ef77 100644
--- a/llvm/test/CodeGen/ARM/popcnt.ll
+++ b/llvm/test/CodeGen/ARM/popcnt.ll
@@ -335,6 +335,7 @@ define i64 @ctpop64(i64 %x) nounwind readnone {
 ; CHECK-NEXT:    .save {r11, lr}
 ; CHECK-NEXT:    push {r11, lr}
 ; CHECK-NEXT:    bl __popcountdi2
+; CHECK-NEXT:    asr r1, r0, #31
 ; CHECK-NEXT:    pop {r11, lr}
 ; CHECK-NEXT:    mov pc, lr
   %count = tail call i64 @llvm.ctpop.i64(i64 %x)
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index d5d167758fb34..576a239967ffb 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -1730,6 +1730,8 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ;
 ; RV64NOZBB-LABEL: test_ctlz_i64_zero_undef:
 ; RV64NOZBB:       # %bb.0:
+; RV64NOZBB-NEXT:    addi sp, sp, -16
+; RV64NOZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64NOZBB-NEXT:    srli a1, a0, 1
 ; RV64NOZBB-NEXT:    or a0, a0, a1
 ; RV64NOZBB-NEXT:    srli a1, a0, 2
@@ -1743,7 +1745,10 @@ define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
 ; RV64NOZBB-NEXT:    srli a1, a0, 32
 ; RV64NOZBB-NEXT:    or a0, a0, a1
 ; RV64NOZBB-NEXT:    not a0, a0
-; RV64NOZBB-NEXT:    tail __popcountdi2
+; RV64NOZBB-NEXT:    call __popcountdi2
+; RV64NOZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    addi sp, sp, 16
+; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctlz_i64_zero_undef:
 ; RV32ZBB:       # %bb.0:
@@ -2058,13 +2063,19 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV32_NOZBB-NEXT:    addi sp, sp, -16
 ; RV32_NOZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32_NOZBB-NEXT:    call __popcountdi2
+; RV32_NOZBB-NEXT:    srai a1, a0, 31
 ; RV32_NOZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    addi sp, sp, 16
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_ctpop_i64:
 ; RV64NOZBB:       # %bb.0:
-; RV64NOZBB-NEXT:    tail __popcountdi2
+; RV64NOZBB-NEXT:    addi sp, sp, -16
+; RV64NOZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    call __popcountdi2
+; RV64NOZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    addi sp, sp, 16
+; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctpop_i64:
 ; RV32ZBB:       # %bb.0:
@@ -2084,13 +2095,19 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
 ; RV32XTHEADBB-NEXT:    addi sp, sp, -16
 ; RV32XTHEADBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32XTHEADBB-NEXT:    call __popcountdi2
+; RV32XTHEADBB-NEXT:    srai a1, a0, 31
 ; RV32XTHEADBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    addi sp, sp, 16
 ; RV32XTHEADBB-NEXT:    ret
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i64:
 ; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    tail __popcountdi2
+; RV64XTHEADBB-NEXT:    addi sp, sp, -16
+; RV64XTHEADBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64XTHEADBB-NEXT:    call __popcountdi2
+; RV64XTHEADBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64XTHEADBB-NEXT:    addi sp, sp, 16
+; RV64XTHEADBB-NEXT:    ret
   %1 = call i64 @llvm.ctpop.i64(i64 %a)
   ret i64 %1
 }
@@ -2104,7 +2121,6 @@ define i128 @test_ctpop_i128(i128 %a) nounwind {
 ; RV32_NOZBB-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; RV32_NOZBB-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32_NOZBB-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
 ; RV32_NOZBB-NEXT:    lw s0, 0(a1)
 ; RV32_NOZBB-NEXT:    lw s1, 4(a1)
 ; RV32_NOZBB-NEXT:    lw a2, 8(a1)
@@ -2113,45 +2129,35 @@ define i128 @test_ctpop_i128(i128 %a) nounwind {
 ; RV32_NOZBB-NEXT:    mv a0, a2
 ; RV32_NOZBB-NEXT:    call __popcountdi2
 ; RV32_NOZBB-NEXT:    mv s3, a0
-; RV32_NOZBB-NEXT:    mv s4, a1
 ; RV32_NOZBB-NEXT:    mv a0, s0
 ; RV32_NOZBB-NEXT:    mv a1, s1
 ; RV32_NOZBB-NEXT:    call __popcountdi2
-; RV32_NOZBB-NEXT:    add a1, a1, s4
-; RV32_NOZBB-NEXT:    add s3, a0, s3
-; RV32_NOZBB-NEXT:    sltu a0, s3, a0
-; RV32_NOZBB-NEXT:    add a0, a1, a0
+; RV32_NOZBB-NEXT:    add a1, a0, s3
+; RV32_NOZBB-NEXT:    sltu a2, a1, a0
+; RV32_NOZBB-NEXT:    srai a3, s3, 31
+; RV32_NOZBB-NEXT:    srai a0, a0, 31
+; RV32_NOZBB-NEXT:    add a0, a0, a3
+; RV32_NOZBB-NEXT:    add a0, a0, a2
 ; RV32_NOZBB-NEXT:    sw zero, 12(s2)
 ; RV32_NOZBB-NEXT:    sw zero, 8(s2)
-; RV32_NOZBB-NEXT:    sw s3, 0(s2)
+; RV32_NOZBB-NEXT:    sw a1, 0(s2)
 ; RV32_NOZBB-NEXT:    sw a0, 4(s2)
 ; RV32_NOZBB-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
 ; RV32_NOZBB-NEXT:    addi sp, sp, 32
 ; RV32_NOZBB-NEXT:    ret
 ;
 ; RV64NOZBB-LABEL: test_ctpop_i128:
 ; RV64NOZBB:       # %bb.0:
-; RV64NOZBB-NEXT:    addi sp, sp, -32
-; RV64NOZBB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64NOZBB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64NOZBB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64NOZBB-NEXT:    mv s0, a0
-; RV64NOZBB-NEXT:    mv a0, a1
-; RV64NOZBB-NEXT:    call __popcountdi2
-; RV64NOZBB-NEXT:    mv s1, a0
-; RV64NOZBB-NEXT:    mv a0, s0
-; RV64NOZBB-NEXT:    call __popcountdi2
-; RV64NOZBB-NEXT:    add a0, a0, s1
-; RV64NOZBB-NEXT:    li a1, 0
-; RV64NOZBB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64NOZBB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64NOZBB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64NOZBB-NEXT:    addi sp, sp, 32
+; RV64NOZBB-NEXT:    addi sp, sp, -16
+; RV64NOZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64NOZBB-NEXT:    call __popcountti2
+; RV64NOZBB-NEXT:    srai a1, a0, 63
+; RV64NOZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64NOZBB-NEXT:    addi sp, sp, 16
 ; RV64NOZBB-NEXT:    ret
 ;
 ; RV32ZBB-LABEL: test_ctpop_i128:
@@ -2190,7 +2196,6 @@ define i128 @test_ctpop_i128(i128 %a) nounwind {
 ; RV32XTHEADBB-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; RV32XTHEADBB-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32XTHEADBB-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
-; RV32XTHEADBB-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
 ; RV32XTHEADBB-NEXT:    lw s0, 0(a1)
 ; RV32XTHEADBB-NEXT:    lw s1, 4(a1)
 ; RV32XTHEADBB-NEXT:    lw a2, 8(a1)
@@ -2199,45 +2204,35 @@ define i128 @test_ctpop_i128(i128 %a) nounwind {
 ; RV32XTHEADBB-NEXT:    mv a0, a2
 ; RV32XTHEADBB-NEXT:    call __popcountdi2
 ; RV32XTHEADBB-NEXT:    mv s3, a0
-; RV32XTHEADBB-NEXT:    mv s4, a1
 ; RV32XTHEADBB-NEXT:    mv a0, s0
 ; RV32XTHEADBB-NEXT:    mv a1, s1
 ; RV32XTHEADBB-NEXT:    call __popcountdi2
-; RV32XTHEADBB-NEXT:    add a1, a1, s4
-; RV32XTHEADBB-NEXT:    add s3, a0, s3
-; RV32XTHEADBB-NEXT:    sltu a0, s3, a0
-; RV32XTHEADBB-NEXT:    add a0, a1, a0
+; RV32XTHEADBB-NEXT:    add a1, a0, s3
+; RV32XTHEADBB-NEXT:    sltu a2, a1, a0
+; RV32XTHEADBB-NEXT:    srai a3, s3, 31
+; RV32XTHEADBB-NEXT:    srai a0, a0, 31
+; RV32XTHEADBB-NEXT:    add a0, a0, a3
+; RV32XTHEADBB-NEXT:    add a0, a0, a2
 ; RV32XTHEADBB-NEXT:    sw zero, 12(s2)
 ; RV32XTHEADBB-NEXT:    sw zero, 8(s2)
-; RV32XTHEADBB-NEXT:    sw s3, 0(s2)
+; RV32XTHEADBB-NEXT:    sw a1, 0(s2)
 ; RV32XTHEADBB-NEXT:    sw a0, 4(s2)
 ; RV32XTHEADBB-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32XTHEADBB-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
 ; RV32XTHEADBB-NEXT:    addi sp, sp, 32
 ; RV32XTHEADBB-NEXT:    ret
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i128:
 ; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    addi sp, sp, -32
-; RV64XTHEADBB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64XTHEADBB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64XTHEADBB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64XTHEADBB-NEXT:    mv s0, a0
-; RV64XTHEADBB-NEXT:    mv a0, a1
-; RV64XTHEADBB-NEXT:    call __popcountdi2
-; RV64XTHEADBB-NEXT:    mv s1, a0
-; RV64XTHEADBB-NEXT:    mv a0, s0
-; RV64XTHEADBB-NEXT:    call __popcountdi2
-; RV64XTHEADBB-NEXT:    add a0, a0, s1
-; RV64XTHEADBB-NEXT:    li a1, 0
-; RV64XTHEADBB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64XTHEADBB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64XTHEADBB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64XTHEADBB-NEXT:    addi sp, sp, 32
+; RV64XTHEADBB-NEXT:    addi sp, sp, -16
+; RV64XTHEADBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64XTHEADBB-NEXT:    call __popcountti2
+; RV64XTHEADBB-NEXT:    srai a1, a0, 63
+; RV64XTHEADBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64XTHEADBB-NEXT:    addi sp, sp, 16
 ; RV64XTHEADBB-NEXT:    ret
   %1 = call i128 @llvm.ctpop.i128(i128 %a)
   ret i128 %1
diff --git a/llvm/test/CodeGen/RISCV/rv32zbb.ll b/llvm/test/CodeGen/RISCV/rv32zbb.ll
index 86e0d6b7b3f9d..a9173836aa42a 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbb.ll
@@ -436,6 +436,7 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __popcountdi2
+; RV32I-NEXT:    srai a1, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -580,14 +581,15 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
 ; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:    call __popcountdi2
 ; RV32I-NEXT:    mv s3, a0
-; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:    srai s4, a0, 31
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __popcountdi2
-; RV32I-NEXT:    sw a1, 12(s2)
+; RV32I-NEXT:    srai a1, a0, 31
 ; RV32I-NEXT:    sw a0, 8(s2)
-; RV32I-NEXT:    sw s4, 4(s2)
 ; RV32I-NEXT:    sw s3, 0(s2)
+; RV32I-NEXT:    sw a1, 12(s2)
+; RV32I-NEXT:    sw s4, 4(s2)
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
index ca04cdd8c8d09..5fed7e6375675 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
@@ -596,7 +596,12 @@ declare i64 @llvm.ctpop.i64(i64)
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    tail __popcountdi2
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: ctpop_i64:
 ; RV64ZBB:       # %bb.0:
@@ -611,22 +616,12 @@ declare i128 @llvm.ctpop.i128(i128)
 define i128 @ctpop_i128(i128 %a) nounwind {
 ; RV64I-LABEL: ctpop_i128:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a0
-; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    call __popcountdi2
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __popcountdi2
-; RV64I-NEXT:    add a0, a0, s1
-; RV64I-NEXT:    li a1, 0
-; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __popcountti2
+; RV64I-NEXT:    srai a1, a0, 63
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: ctpop_i128:
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 0ced55b0dd63c..2915f112499b6 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -804,7 +804,12 @@ declare i64 @llvm.ctpop.i64(i64)
 define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    tail __popcountdi2
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __popcountdi2
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: ctpop_i64:
 ; RV64ZBB:       # %bb.0:
@@ -895,22 +900,12 @@ declare i128 @llvm.ctpop.i128(i128)
 define i128 @ctpop_i128(i128 %a) nounwind {
 ; RV64I-LABEL: ctpop_i128:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    mv s0, a0
-; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    call __popcountdi2
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __popcountdi2
-; RV64I-NEXT:    add a0, a0, s1
-; RV64I-NEXT:    li a1, 0
-; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __popcountti2
+; RV64I-NEXT:    srai a1, a0, 63
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBB-LABEL: ctpop_i128:
diff --git a/llvm/test/CodeGen/Thumb2/mve-ctpop.ll b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
index 670568f50a6fe..9f89b2c495659 100644
--- a/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-ctpop.ll
@@ -12,9 +12,10 @@ define arm_aapcs_vfpcc <2 x i64> @ctpop_2i64_t(<2 x i64> %src){
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    bl __popcountdi2
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    asrs r5, r4, #31
 ; CHECK-NEXT:    bl __popcountdi2
+; CHECK-NEXT:    asrs r1, r0, #31
 ; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
 ; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}



More information about the llvm-commits mailing list