[llvm-branch-commits] [llvm] dcf1296 - Revert "[SDag][ARM][RISCV] Allow lowering CTPOP into a libcall (#99752)"
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 2 12:32:41 PDT 2024
Author: Sergei Barannikov
Date: 2024-08-02T22:32:37+03:00
New Revision: dcf1296a5e76ef0a0a297284f03a8d4b470b8376
URL: https://github.com/llvm/llvm-project/commit/dcf1296a5e76ef0a0a297284f03a8d4b470b8376
DIFF: https://github.com/llvm/llvm-project/commit/dcf1296a5e76ef0a0a297284f03a8d4b470b8376.diff
LOG: Revert "[SDag][ARM][RISCV] Allow lowering CTPOP into a libcall (#99752)"
This reverts commit 92e18ffd803365c64910760ba20278f875d93681.
Added:
Modified:
llvm/include/llvm/IR/RuntimeLibcalls.def
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
llvm/test/CodeGen/RISCV/pr56457.ll
llvm/test/CodeGen/RISCV/pr95271.ll
llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
llvm/test/CodeGen/RISCV/rv32zbb.ll
llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
llvm/test/CodeGen/RISCV/rv64zbb.ll
llvm/test/CodeGen/RISCV/sextw-removal.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 3dd75622b8e43..89aaf6d1ad83f 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -85,9 +85,6 @@ HANDLE_LIBCALL(NEG_I64, "__negdi2")
HANDLE_LIBCALL(CTLZ_I32, "__clzsi2")
HANDLE_LIBCALL(CTLZ_I64, "__clzdi2")
HANDLE_LIBCALL(CTLZ_I128, "__clzti2")
-HANDLE_LIBCALL(CTPOP_I32, "__popcountsi2")
-HANDLE_LIBCALL(CTPOP_I64, "__popcountdi2")
-HANDLE_LIBCALL(CTPOP_I128, "__popcountti2")
// Floating-point
HANDLE_LIBCALL(ADD_F32, "__addsf3")
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index c91a2360d1599..bdb7917073020 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -140,9 +140,12 @@ class SelectionDAGLegalize {
RTLIB::Libcall Call_F128,
RTLIB::Libcall Call_PPCF128,
SmallVectorImpl<SDValue> &Results);
- SDValue ExpandIntLibCall(SDNode *Node, bool IsSigned, RTLIB::Libcall Call_I8,
- RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
- RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
+ SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
+ RTLIB::Libcall Call_I8,
+ RTLIB::Libcall Call_I16,
+ RTLIB::Libcall Call_I32,
+ RTLIB::Libcall Call_I64,
+ RTLIB::Libcall Call_I128);
void ExpandArgFPLibCall(SDNode *Node,
RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
@@ -2206,7 +2209,7 @@ void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
ExpandFPLibCall(Node, LC, Results);
}
-SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode *Node, bool IsSigned,
+SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
RTLIB::Libcall Call_I8,
RTLIB::Libcall Call_I16,
RTLIB::Libcall Call_I32,
@@ -2221,9 +2224,7 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode *Node, bool IsSigned,
case MVT::i64: LC = Call_I64; break;
case MVT::i128: LC = Call_I128; break;
}
- assert(LC != RTLIB::UNKNOWN_LIBCALL &&
- "LibCall explicitly requested, but not available");
- return ExpandLibCall(LC, Node, IsSigned).first;
+ return ExpandLibCall(LC, Node, isSigned).first;
}
/// Expand the node to a libcall based on first argument type (for instance
@@ -4999,16 +5000,19 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::MUL_I64, RTLIB::MUL_I128));
break;
case ISD::CTLZ_ZERO_UNDEF:
- Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
- RTLIB::UNKNOWN_LIBCALL,
- RTLIB::UNKNOWN_LIBCALL, RTLIB::CTLZ_I32,
- RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
- break;
- case ISD::CTPOP:
- Results.push_back(ExpandIntLibCall(Node, /*IsSigned=*/false,
- RTLIB::UNKNOWN_LIBCALL,
- RTLIB::UNKNOWN_LIBCALL, RTLIB::CTPOP_I32,
- RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
+ switch (Node->getSimpleValueType(0).SimpleTy) {
+ default:
+ llvm_unreachable("LibCall explicitly requested, but not available");
+ case MVT::i32:
+ Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
+ break;
+ case MVT::i64:
+ Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
+ break;
+ case MVT::i128:
+ Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
+ break;
+ }
break;
case ISD::RESET_FPENV: {
// It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index d00785025bac5..b1ada66aa9aeb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3850,33 +3850,15 @@ void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
Hi = DAG.getConstant(0, dl, NVT);
}
-void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, SDValue &Lo, SDValue &Hi) {
- SDValue Op = N->getOperand(0);
- EVT VT = N->getValueType(0);
- SDLoc DL(N);
-
- if (TLI.getOperationAction(ISD::CTPOP, VT) == TargetLoweringBase::LibCall) {
- RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
- if (VT == MVT::i32)
- LC = RTLIB::CTPOP_I32;
- else if (VT == MVT::i64)
- LC = RTLIB::CTPOP_I64;
- else if (VT == MVT::i128)
- LC = RTLIB::CTPOP_I128;
- assert(LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC) &&
- "LibCall explicitly requested, but not available");
- TargetLowering::MakeLibCallOptions CallOptions;
- SDValue Res = TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, DL).first;
- SplitInteger(Res, Lo, Hi);
- return;
- }
-
+void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
+ SDValue &Lo, SDValue &Hi) {
+ SDLoc dl(N);
// ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
- GetExpandedInteger(Op, Lo, Hi);
+ GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();
- Lo = DAG.getNode(ISD::ADD, DL, NVT, DAG.getNode(ISD::CTPOP, DL, NVT, Lo),
- DAG.getNode(ISD::CTPOP, DL, NVT, Hi));
- Hi = DAG.getConstant(0, DL, NVT);
+ Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
+ DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
+ Hi = DAG.getConstant(0, dl, NVT);
}
void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 8ab3103fda23f..83aadcfd241e9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9171,9 +9171,8 @@ SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
return SDValue();
- // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
- // to be expanded or converted to a libcall.
- if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(ISD::CTPOP, VT) &&
+ // Emit Table Lookup if ISD::CTLZ and ISD::CTPOP are not legal.
+ if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) &&
!isOperationLegal(ISD::CTLZ, VT))
if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt))
return V;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4a2193c8d5328..9ee60b9db2837 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -393,10 +393,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
}
} else {
- setOperationAction(ISD::CTTZ, XLenVT, Expand);
- if (!Subtarget.is64Bit())
- setOperationAction(ISD::CTPOP, MVT::i32, LibCall);
- setOperationAction(ISD::CTPOP, MVT::i64, LibCall);
+ setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand);
if (RV64LegalI32 && Subtarget.is64Bit())
setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand);
}
diff --git a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
index c0ecc63b82dca..380f65b19b8fa 100644
--- a/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/int-bit-manip.ll
@@ -159,7 +159,7 @@ define void @bitreverse() {
define void @ctpop() {
; NOZVBB-LABEL: 'ctpop'
-; NOZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
+; NOZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %2 = call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %3 = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %4 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> undef)
@@ -169,7 +169,7 @@ define void @ctpop() {
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %8 = call <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %9 = call <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
-; NOZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
+; NOZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %12 = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %13 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %14 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> undef)
@@ -179,7 +179,7 @@ define void @ctpop() {
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
-; NOZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
+; NOZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %22 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %23 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
@@ -189,7 +189,7 @@ define void @ctpop() {
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
-; NOZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
+; NOZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %32 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
@@ -202,7 +202,7 @@ define void @ctpop() {
; NOZVBB-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; ZVBB-LABEL: 'ctpop'
-; ZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
+; ZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = call i8 @llvm.ctpop.i8(i8 undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> undef)
@@ -212,7 +212,7 @@ define void @ctpop() {
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i8> @llvm.ctpop.nxv4i8(<vscale x 4 x i8> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i8> @llvm.ctpop.nxv8i8(<vscale x 8 x i8> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> undef)
-; ZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
+; ZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = call i16 @llvm.ctpop.i16(i16 undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> undef)
@@ -222,7 +222,7 @@ define void @ctpop() {
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i16> @llvm.ctpop.nxv4i16(<vscale x 4 x i16> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = call <vscale x 16 x i16> @llvm.ctpop.nxv16i16(<vscale x 16 x i16> undef)
-; ZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
+; ZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %21 = call i32 @llvm.ctpop.i32(i32 undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> undef)
@@ -232,7 +232,7 @@ define void @ctpop() {
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %29 = call <vscale x 8 x i32> @llvm.ctpop.nxv8i32(<vscale x 8 x i32> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = call <vscale x 16 x i32> @llvm.ctpop.nxv16i32(<vscale x 16 x i32> undef)
-; ZVBB-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
+; ZVBB-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %31 = call i64 @llvm.ctpop.i64(i64 undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %32 = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %33 = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> undef)
; ZVBB-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %34 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> undef)
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 9123017918094..8caa64c9572ce 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -1156,30 +1156,46 @@ define i16 @test_ctlz_i16(i16 %a) nounwind {
}
define i32 @test_ctlz_i32(i32 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctlz_i32:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: beqz a0, .LBB10_2
-; RV32_NOZBB-NEXT: # %bb.1: # %cond.false
-; RV32_NOZBB-NEXT: addi sp, sp, -16
-; RV32_NOZBB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: srli a1, a0, 1
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: call __popcountsi2
-; RV32_NOZBB-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: addi sp, sp, 16
-; RV32_NOZBB-NEXT: ret
-; RV32_NOZBB-NEXT: .LBB10_2:
-; RV32_NOZBB-NEXT: li a0, 32
-; RV32_NOZBB-NEXT: ret
+; RV32I-LABEL: test_ctlz_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: beqz a0, .LBB10_2
+; RV32I-NEXT: # %bb.1: # %cond.false
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB10_2:
+; RV32I-NEXT: li a0, 32
+; RV32I-NEXT: ret
;
; RV64I-LABEL: test_ctlz_i32:
; RV64I: # %bb.0:
@@ -1223,6 +1239,46 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
; RV64I-NEXT: li a0, 32
; RV64I-NEXT: ret
;
+; RV32M-LABEL: test_ctlz_i32:
+; RV32M: # %bb.0:
+; RV32M-NEXT: beqz a0, .LBB10_2
+; RV32M-NEXT: # %bb.1: # %cond.false
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: lui a2, 349525
+; RV32M-NEXT: addi a2, a2, 1365
+; RV32M-NEXT: and a1, a1, a2
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: lui a1, 209715
+; RV32M-NEXT: addi a1, a1, 819
+; RV32M-NEXT: and a2, a0, a1
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: add a0, a2, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: lui a1, 61681
+; RV32M-NEXT: addi a1, a1, -241
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: lui a1, 4112
+; RV32M-NEXT: addi a1, a1, 257
+; RV32M-NEXT: mul a0, a0, a1
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: ret
+; RV32M-NEXT: .LBB10_2:
+; RV32M-NEXT: li a0, 32
+; RV32M-NEXT: ret
+;
; RV64M-LABEL: test_ctlz_i32:
; RV64M: # %bb.0:
; RV64M-NEXT: sext.w a1, a0
@@ -1290,75 +1346,240 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
}
define i64 @test_ctlz_i64(i64 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctlz_i64:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: addi sp, sp, -16
-; RV32_NOZBB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: mv s1, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 1
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: call __popcountsi2
-; RV32_NOZBB-NEXT: mv s0, a0
-; RV32_NOZBB-NEXT: srli a0, s1, 1
-; RV32_NOZBB-NEXT: or a0, s1, a0
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: call __popcountsi2
-; RV32_NOZBB-NEXT: bnez s1, .LBB11_2
-; RV32_NOZBB-NEXT: # %bb.1:
-; RV32_NOZBB-NEXT: addi a0, s0, 32
-; RV32_NOZBB-NEXT: .LBB11_2:
-; RV32_NOZBB-NEXT: li a1, 0
-; RV32_NOZBB-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: addi sp, sp, 16
-; RV32_NOZBB-NEXT: ret
+; RV32I-LABEL: test_ctlz_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a4, a2, 1365
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a3, a2, 819
+; RV32I-NEXT: lui a2, 61681
+; RV32I-NEXT: addi a2, a2, -241
+; RV32I-NEXT: bnez a1, .LBB11_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: addi a0, a0, 32
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB11_2:
+; RV32I-NEXT: srli a0, a1, 1
+; RV32I-NEXT: or a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
;
-; RV64NOZBB-LABEL: test_ctlz_i64:
-; RV64NOZBB: # %bb.0:
-; RV64NOZBB-NEXT: beqz a0, .LBB11_2
-; RV64NOZBB-NEXT: # %bb.1: # %cond.false
-; RV64NOZBB-NEXT: addi sp, sp, -16
-; RV64NOZBB-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
-; RV64NOZBB-NEXT: srli a1, a0, 1
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 2
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 4
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 8
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 16
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 32
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: not a0, a0
-; RV64NOZBB-NEXT: call __popcountdi2
-; RV64NOZBB-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64NOZBB-NEXT: addi sp, sp, 16
-; RV64NOZBB-NEXT: ret
-; RV64NOZBB-NEXT: .LBB11_2:
-; RV64NOZBB-NEXT: li a0, 64
-; RV64NOZBB-NEXT: ret
+; RV64I-LABEL: test_ctlz_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: beqz a0, .LBB11_2
+; RV64I-NEXT: # %bb.1: # %cond.false
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 2
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 8
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 16
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 32
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: not a0, a0
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: ret
+; RV64I-NEXT: .LBB11_2:
+; RV64I-NEXT: li a0, 64
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: test_ctlz_i64:
+; RV32M: # %bb.0:
+; RV32M-NEXT: lui a2, 349525
+; RV32M-NEXT: addi a5, a2, 1365
+; RV32M-NEXT: lui a2, 209715
+; RV32M-NEXT: addi a4, a2, 819
+; RV32M-NEXT: lui a2, 61681
+; RV32M-NEXT: addi a2, a2, -241
+; RV32M-NEXT: lui a3, 4112
+; RV32M-NEXT: addi a3, a3, 257
+; RV32M-NEXT: bnez a1, .LBB11_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: and a1, a1, a5
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: and a1, a0, a4
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a4
+; RV32M-NEXT: add a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: and a0, a0, a2
+; RV32M-NEXT: mul a0, a0, a3
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: addi a0, a0, 32
+; RV32M-NEXT: li a1, 0
+; RV32M-NEXT: ret
+; RV32M-NEXT: .LBB11_2:
+; RV32M-NEXT: srli a0, a1, 1
+; RV32M-NEXT: or a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: and a1, a1, a5
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: and a1, a0, a4
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a4
+; RV32M-NEXT: add a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: and a0, a0, a2
+; RV32M-NEXT: mul a0, a0, a3
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: li a1, 0
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: test_ctlz_i64:
+; RV64M: # %bb.0:
+; RV64M-NEXT: beqz a0, .LBB11_2
+; RV64M-NEXT: # %bb.1: # %cond.false
+; RV64M-NEXT: srli a1, a0, 1
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 2
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 4
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 8
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 16
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 32
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: not a0, a0
+; RV64M-NEXT: srli a1, a0, 1
+; RV64M-NEXT: lui a2, 349525
+; RV64M-NEXT: addiw a2, a2, 1365
+; RV64M-NEXT: slli a3, a2, 32
+; RV64M-NEXT: add a2, a2, a3
+; RV64M-NEXT: and a1, a1, a2
+; RV64M-NEXT: sub a0, a0, a1
+; RV64M-NEXT: lui a1, 209715
+; RV64M-NEXT: addiw a1, a1, 819
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a2, a0, a1
+; RV64M-NEXT: srli a0, a0, 2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: add a0, a2, a0
+; RV64M-NEXT: srli a1, a0, 4
+; RV64M-NEXT: add a0, a0, a1
+; RV64M-NEXT: lui a1, 61681
+; RV64M-NEXT: addiw a1, a1, -241
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: lui a1, 4112
+; RV64M-NEXT: addiw a1, a1, 257
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: mul a0, a0, a1
+; RV64M-NEXT: srli a0, a0, 56
+; RV64M-NEXT: ret
+; RV64M-NEXT: .LBB11_2:
+; RV64M-NEXT: li a0, 64
+; RV64M-NEXT: ret
;
; RV32ZBB-LABEL: test_ctlz_i64:
; RV32ZBB: # %bb.0:
@@ -1572,20 +1793,41 @@ define i16 @test_ctlz_i16_zero_undef(i16 %a) nounwind {
}
define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctlz_i32_zero_undef:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: srli a1, a0, 1
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: tail __popcountsi2
+; RV32I-LABEL: test_ctlz_i32_zero_undef:
+; RV32I: # %bb.0:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: ret
;
; RV64I-LABEL: test_ctlz_i32_zero_undef:
; RV64I: # %bb.0:
@@ -1623,6 +1865,41 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
; RV64I-NEXT: srliw a0, a0, 24
; RV64I-NEXT: ret
;
+; RV32M-LABEL: test_ctlz_i32_zero_undef:
+; RV32M: # %bb.0:
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: lui a2, 349525
+; RV32M-NEXT: addi a2, a2, 1365
+; RV32M-NEXT: and a1, a1, a2
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: lui a1, 209715
+; RV32M-NEXT: addi a1, a1, 819
+; RV32M-NEXT: and a2, a0, a1
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: add a0, a2, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: lui a1, 61681
+; RV32M-NEXT: addi a1, a1, -241
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: lui a1, 4112
+; RV32M-NEXT: addi a1, a1, 257
+; RV32M-NEXT: mul a0, a0, a1
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: ret
+;
; RV64M-LABEL: test_ctlz_i32_zero_undef:
; RV64M: # %bb.0:
; RV64M-NEXT: srliw a1, a0, 1
@@ -1684,65 +1961,230 @@ define i32 @test_ctlz_i32_zero_undef(i32 %a) nounwind {
}
define i64 @test_ctlz_i64_zero_undef(i64 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctlz_i64_zero_undef:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: addi sp, sp, -16
-; RV32_NOZBB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: mv s1, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 1
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: call __popcountsi2
-; RV32_NOZBB-NEXT: mv s0, a0
-; RV32_NOZBB-NEXT: srli a0, s1, 1
-; RV32_NOZBB-NEXT: or a0, s1, a0
-; RV32_NOZBB-NEXT: srli a1, a0, 2
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 4
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 8
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: srli a1, a0, 16
-; RV32_NOZBB-NEXT: or a0, a0, a1
-; RV32_NOZBB-NEXT: not a0, a0
-; RV32_NOZBB-NEXT: call __popcountsi2
-; RV32_NOZBB-NEXT: bnez s1, .LBB15_2
-; RV32_NOZBB-NEXT: # %bb.1:
-; RV32_NOZBB-NEXT: addi a0, s0, 32
-; RV32_NOZBB-NEXT: .LBB15_2:
-; RV32_NOZBB-NEXT: li a1, 0
-; RV32_NOZBB-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: addi sp, sp, 16
-; RV32_NOZBB-NEXT: ret
+; RV32I-LABEL: test_ctlz_i64_zero_undef:
+; RV32I: # %bb.0:
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a4, a2, 1365
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a3, a2, 819
+; RV32I-NEXT: lui a2, 61681
+; RV32I-NEXT: addi a2, a2, -241
+; RV32I-NEXT: bnez a1, .LBB15_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: addi a0, a0, 32
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB15_2:
+; RV32I-NEXT: srli a0, a1, 1
+; RV32I-NEXT: or a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 2
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 8
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: srli a1, a0, 16
+; RV32I-NEXT: or a0, a0, a1
+; RV32I-NEXT: not a0, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
;
-; RV64NOZBB-LABEL: test_ctlz_i64_zero_undef:
-; RV64NOZBB: # %bb.0:
-; RV64NOZBB-NEXT: srli a1, a0, 1
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 2
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 4
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 8
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 16
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: srli a1, a0, 32
-; RV64NOZBB-NEXT: or a0, a0, a1
-; RV64NOZBB-NEXT: not a0, a0
-; RV64NOZBB-NEXT: tail __popcountdi2
+; RV64I-LABEL: test_ctlz_i64_zero_undef:
+; RV64I: # %bb.0:
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 2
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 8
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 16
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: srli a1, a0, 32
+; RV64I-NEXT: or a0, a0, a1
+; RV64I-NEXT: not a0, a0
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: test_ctlz_i64_zero_undef:
+; RV32M: # %bb.0:
+; RV32M-NEXT: lui a2, 349525
+; RV32M-NEXT: addi a5, a2, 1365
+; RV32M-NEXT: lui a2, 209715
+; RV32M-NEXT: addi a4, a2, 819
+; RV32M-NEXT: lui a2, 61681
+; RV32M-NEXT: addi a2, a2, -241
+; RV32M-NEXT: lui a3, 4112
+; RV32M-NEXT: addi a3, a3, 257
+; RV32M-NEXT: bnez a1, .LBB15_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: and a1, a1, a5
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: and a1, a0, a4
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a4
+; RV32M-NEXT: add a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: and a0, a0, a2
+; RV32M-NEXT: mul a0, a0, a3
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: addi a0, a0, 32
+; RV32M-NEXT: li a1, 0
+; RV32M-NEXT: ret
+; RV32M-NEXT: .LBB15_2:
+; RV32M-NEXT: srli a0, a1, 1
+; RV32M-NEXT: or a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 2
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 8
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: srli a1, a0, 16
+; RV32M-NEXT: or a0, a0, a1
+; RV32M-NEXT: not a0, a0
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: and a1, a1, a5
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: and a1, a0, a4
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a4
+; RV32M-NEXT: add a0, a1, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: and a0, a0, a2
+; RV32M-NEXT: mul a0, a0, a3
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: li a1, 0
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: test_ctlz_i64_zero_undef:
+; RV64M: # %bb.0:
+; RV64M-NEXT: srli a1, a0, 1
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 2
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 4
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 8
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 16
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: srli a1, a0, 32
+; RV64M-NEXT: or a0, a0, a1
+; RV64M-NEXT: not a0, a0
+; RV64M-NEXT: srli a1, a0, 1
+; RV64M-NEXT: lui a2, 349525
+; RV64M-NEXT: addiw a2, a2, 1365
+; RV64M-NEXT: slli a3, a2, 32
+; RV64M-NEXT: add a2, a2, a3
+; RV64M-NEXT: and a1, a1, a2
+; RV64M-NEXT: sub a0, a0, a1
+; RV64M-NEXT: lui a1, 209715
+; RV64M-NEXT: addiw a1, a1, 819
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a2, a0, a1
+; RV64M-NEXT: srli a0, a0, 2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: add a0, a2, a0
+; RV64M-NEXT: srli a1, a0, 4
+; RV64M-NEXT: add a0, a0, a1
+; RV64M-NEXT: lui a1, 61681
+; RV64M-NEXT: addiw a1, a1, -241
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: lui a1, 4112
+; RV64M-NEXT: addiw a1, a1, 257
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: mul a0, a0, a1
+; RV64M-NEXT: srli a0, a0, 56
+; RV64M-NEXT: ret
;
; RV32ZBB-LABEL: test_ctlz_i64_zero_undef:
; RV32ZBB: # %bb.0:
@@ -1956,9 +2398,30 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
}
define i32 @test_ctpop_i32(i32 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctpop_i32:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: tail __popcountsi2
+; RV32I-LABEL: test_ctpop_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: ret
;
; RV64I-LABEL: test_ctpop_i32:
; RV64I: # %bb.0:
@@ -1985,6 +2448,30 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
; RV64I-NEXT: srliw a0, a0, 24
; RV64I-NEXT: ret
;
+; RV32M-LABEL: test_ctpop_i32:
+; RV32M: # %bb.0:
+; RV32M-NEXT: srli a1, a0, 1
+; RV32M-NEXT: lui a2, 349525
+; RV32M-NEXT: addi a2, a2, 1365
+; RV32M-NEXT: and a1, a1, a2
+; RV32M-NEXT: sub a0, a0, a1
+; RV32M-NEXT: lui a1, 209715
+; RV32M-NEXT: addi a1, a1, 819
+; RV32M-NEXT: and a2, a0, a1
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: add a0, a2, a0
+; RV32M-NEXT: srli a1, a0, 4
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: lui a1, 61681
+; RV32M-NEXT: addi a1, a1, -241
+; RV32M-NEXT: and a0, a0, a1
+; RV32M-NEXT: lui a1, 4112
+; RV32M-NEXT: addi a1, a1, 257
+; RV32M-NEXT: mul a0, a0, a1
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: ret
+;
; RV64M-LABEL: test_ctpop_i32:
; RV64M: # %bb.0:
; RV64M-NEXT: srli a1, a0, 1
@@ -2021,7 +2508,28 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
;
; RV32XTHEADBB-LABEL: test_ctpop_i32:
; RV32XTHEADBB: # %bb.0:
-; RV32XTHEADBB-NEXT: tail __popcountsi2
+; RV32XTHEADBB-NEXT: srli a1, a0, 1
+; RV32XTHEADBB-NEXT: lui a2, 349525
+; RV32XTHEADBB-NEXT: addi a2, a2, 1365
+; RV32XTHEADBB-NEXT: and a1, a1, a2
+; RV32XTHEADBB-NEXT: sub a0, a0, a1
+; RV32XTHEADBB-NEXT: lui a1, 209715
+; RV32XTHEADBB-NEXT: addi a1, a1, 819
+; RV32XTHEADBB-NEXT: and a2, a0, a1
+; RV32XTHEADBB-NEXT: srli a0, a0, 2
+; RV32XTHEADBB-NEXT: and a0, a0, a1
+; RV32XTHEADBB-NEXT: add a0, a2, a0
+; RV32XTHEADBB-NEXT: srli a1, a0, 4
+; RV32XTHEADBB-NEXT: add a0, a0, a1
+; RV32XTHEADBB-NEXT: lui a1, 61681
+; RV32XTHEADBB-NEXT: addi a1, a1, -241
+; RV32XTHEADBB-NEXT: and a0, a0, a1
+; RV32XTHEADBB-NEXT: slli a1, a0, 8
+; RV32XTHEADBB-NEXT: add a0, a0, a1
+; RV32XTHEADBB-NEXT: slli a1, a0, 16
+; RV32XTHEADBB-NEXT: add a0, a0, a1
+; RV32XTHEADBB-NEXT: srli a0, a0, 24
+; RV32XTHEADBB-NEXT: ret
;
; RV64XTHEADBB-LABEL: test_ctpop_i32:
; RV64XTHEADBB: # %bb.0:
@@ -2052,18 +2560,150 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
}
define i64 @test_ctpop_i64(i64 %a) nounwind {
-; RV32_NOZBB-LABEL: test_ctpop_i64:
-; RV32_NOZBB: # %bb.0:
-; RV32_NOZBB-NEXT: addi sp, sp, -16
-; RV32_NOZBB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32_NOZBB-NEXT: call __popcountdi2
-; RV32_NOZBB-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32_NOZBB-NEXT: addi sp, sp, 16
-; RV32_NOZBB-NEXT: ret
+; RV32I-LABEL: test_ctpop_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: srli a2, a1, 1
+; RV32I-NEXT: lui a3, 349525
+; RV32I-NEXT: addi a3, a3, 1365
+; RV32I-NEXT: and a2, a2, a3
+; RV32I-NEXT: sub a1, a1, a2
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a2, a2, 819
+; RV32I-NEXT: and a4, a1, a2
+; RV32I-NEXT: srli a1, a1, 2
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: add a1, a4, a1
+; RV32I-NEXT: srli a4, a1, 4
+; RV32I-NEXT: add a1, a1, a4
+; RV32I-NEXT: lui a4, 61681
+; RV32I-NEXT: addi a4, a4, -241
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: slli a5, a1, 8
+; RV32I-NEXT: add a1, a1, a5
+; RV32I-NEXT: slli a5, a1, 16
+; RV32I-NEXT: add a1, a1, a5
+; RV32I-NEXT: srli a1, a1, 24
+; RV32I-NEXT: srli a5, a0, 1
+; RV32I-NEXT: and a3, a5, a3
+; RV32I-NEXT: sub a0, a0, a3
+; RV32I-NEXT: and a3, a0, a2
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: add a0, a3, a0
+; RV32I-NEXT: srli a2, a0, 4
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: and a0, a0, a4
+; RV32I-NEXT: slli a2, a0, 8
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: slli a2, a0, 16
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
;
-; RV64NOZBB-LABEL: test_ctpop_i64:
-; RV64NOZBB: # %bb.0:
-; RV64NOZBB-NEXT: tail __popcountdi2
+; RV64I-LABEL: test_ctpop_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: test_ctpop_i64:
+; RV32M: # %bb.0:
+; RV32M-NEXT: srli a2, a1, 1
+; RV32M-NEXT: lui a3, 349525
+; RV32M-NEXT: addi a3, a3, 1365
+; RV32M-NEXT: and a2, a2, a3
+; RV32M-NEXT: sub a1, a1, a2
+; RV32M-NEXT: lui a2, 209715
+; RV32M-NEXT: addi a2, a2, 819
+; RV32M-NEXT: and a4, a1, a2
+; RV32M-NEXT: srli a1, a1, 2
+; RV32M-NEXT: and a1, a1, a2
+; RV32M-NEXT: add a1, a4, a1
+; RV32M-NEXT: srli a4, a1, 4
+; RV32M-NEXT: add a1, a1, a4
+; RV32M-NEXT: lui a4, 61681
+; RV32M-NEXT: addi a4, a4, -241
+; RV32M-NEXT: and a1, a1, a4
+; RV32M-NEXT: lui a5, 4112
+; RV32M-NEXT: addi a5, a5, 257
+; RV32M-NEXT: mul a1, a1, a5
+; RV32M-NEXT: srli a1, a1, 24
+; RV32M-NEXT: srli a6, a0, 1
+; RV32M-NEXT: and a3, a6, a3
+; RV32M-NEXT: sub a0, a0, a3
+; RV32M-NEXT: and a3, a0, a2
+; RV32M-NEXT: srli a0, a0, 2
+; RV32M-NEXT: and a0, a0, a2
+; RV32M-NEXT: add a0, a3, a0
+; RV32M-NEXT: srli a2, a0, 4
+; RV32M-NEXT: add a0, a0, a2
+; RV32M-NEXT: and a0, a0, a4
+; RV32M-NEXT: mul a0, a0, a5
+; RV32M-NEXT: srli a0, a0, 24
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: li a1, 0
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: test_ctpop_i64:
+; RV64M: # %bb.0:
+; RV64M-NEXT: srli a1, a0, 1
+; RV64M-NEXT: lui a2, 349525
+; RV64M-NEXT: addiw a2, a2, 1365
+; RV64M-NEXT: slli a3, a2, 32
+; RV64M-NEXT: add a2, a2, a3
+; RV64M-NEXT: and a1, a1, a2
+; RV64M-NEXT: sub a0, a0, a1
+; RV64M-NEXT: lui a1, 209715
+; RV64M-NEXT: addiw a1, a1, 819
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a2, a0, a1
+; RV64M-NEXT: srli a0, a0, 2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: add a0, a2, a0
+; RV64M-NEXT: srli a1, a0, 4
+; RV64M-NEXT: add a0, a0, a1
+; RV64M-NEXT: lui a1, 61681
+; RV64M-NEXT: addiw a1, a1, -241
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: and a0, a0, a1
+; RV64M-NEXT: lui a1, 4112
+; RV64M-NEXT: addiw a1, a1, 257
+; RV64M-NEXT: slli a2, a1, 32
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: mul a0, a0, a1
+; RV64M-NEXT: srli a0, a0, 56
+; RV64M-NEXT: ret
;
; RV32ZBB-LABEL: test_ctpop_i64:
; RV32ZBB: # %bb.0:
@@ -2080,16 +2720,78 @@ define i64 @test_ctpop_i64(i64 %a) nounwind {
;
; RV32XTHEADBB-LABEL: test_ctpop_i64:
; RV32XTHEADBB: # %bb.0:
-; RV32XTHEADBB-NEXT: addi sp, sp, -16
-; RV32XTHEADBB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32XTHEADBB-NEXT: call __popcountdi2
-; RV32XTHEADBB-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32XTHEADBB-NEXT: addi sp, sp, 16
+; RV32XTHEADBB-NEXT: srli a2, a1, 1
+; RV32XTHEADBB-NEXT: lui a3, 349525
+; RV32XTHEADBB-NEXT: addi a3, a3, 1365
+; RV32XTHEADBB-NEXT: and a2, a2, a3
+; RV32XTHEADBB-NEXT: sub a1, a1, a2
+; RV32XTHEADBB-NEXT: lui a2, 209715
+; RV32XTHEADBB-NEXT: addi a2, a2, 819
+; RV32XTHEADBB-NEXT: and a4, a1, a2
+; RV32XTHEADBB-NEXT: srli a1, a1, 2
+; RV32XTHEADBB-NEXT: and a1, a1, a2
+; RV32XTHEADBB-NEXT: add a1, a4, a1
+; RV32XTHEADBB-NEXT: srli a4, a1, 4
+; RV32XTHEADBB-NEXT: add a1, a1, a4
+; RV32XTHEADBB-NEXT: lui a4, 61681
+; RV32XTHEADBB-NEXT: addi a4, a4, -241
+; RV32XTHEADBB-NEXT: and a1, a1, a4
+; RV32XTHEADBB-NEXT: slli a5, a1, 8
+; RV32XTHEADBB-NEXT: add a1, a1, a5
+; RV32XTHEADBB-NEXT: slli a5, a1, 16
+; RV32XTHEADBB-NEXT: add a1, a1, a5
+; RV32XTHEADBB-NEXT: srli a1, a1, 24
+; RV32XTHEADBB-NEXT: srli a5, a0, 1
+; RV32XTHEADBB-NEXT: and a3, a5, a3
+; RV32XTHEADBB-NEXT: sub a0, a0, a3
+; RV32XTHEADBB-NEXT: and a3, a0, a2
+; RV32XTHEADBB-NEXT: srli a0, a0, 2
+; RV32XTHEADBB-NEXT: and a0, a0, a2
+; RV32XTHEADBB-NEXT: add a0, a3, a0
+; RV32XTHEADBB-NEXT: srli a2, a0, 4
+; RV32XTHEADBB-NEXT: add a0, a0, a2
+; RV32XTHEADBB-NEXT: and a0, a0, a4
+; RV32XTHEADBB-NEXT: slli a2, a0, 8
+; RV32XTHEADBB-NEXT: add a0, a0, a2
+; RV32XTHEADBB-NEXT: slli a2, a0, 16
+; RV32XTHEADBB-NEXT: add a0, a0, a2
+; RV32XTHEADBB-NEXT: srli a0, a0, 24
+; RV32XTHEADBB-NEXT: add a0, a0, a1
+; RV32XTHEADBB-NEXT: li a1, 0
; RV32XTHEADBB-NEXT: ret
;
; RV64XTHEADBB-LABEL: test_ctpop_i64:
; RV64XTHEADBB: # %bb.0:
-; RV64XTHEADBB-NEXT: tail __popcountdi2
+; RV64XTHEADBB-NEXT: srli a1, a0, 1
+; RV64XTHEADBB-NEXT: lui a2, 349525
+; RV64XTHEADBB-NEXT: addiw a2, a2, 1365
+; RV64XTHEADBB-NEXT: slli a3, a2, 32
+; RV64XTHEADBB-NEXT: add a2, a2, a3
+; RV64XTHEADBB-NEXT: and a1, a1, a2
+; RV64XTHEADBB-NEXT: sub a0, a0, a1
+; RV64XTHEADBB-NEXT: lui a1, 209715
+; RV64XTHEADBB-NEXT: addiw a1, a1, 819
+; RV64XTHEADBB-NEXT: slli a2, a1, 32
+; RV64XTHEADBB-NEXT: add a1, a1, a2
+; RV64XTHEADBB-NEXT: and a2, a0, a1
+; RV64XTHEADBB-NEXT: srli a0, a0, 2
+; RV64XTHEADBB-NEXT: and a0, a0, a1
+; RV64XTHEADBB-NEXT: add a0, a2, a0
+; RV64XTHEADBB-NEXT: srli a1, a0, 4
+; RV64XTHEADBB-NEXT: add a0, a0, a1
+; RV64XTHEADBB-NEXT: lui a1, 61681
+; RV64XTHEADBB-NEXT: addiw a1, a1, -241
+; RV64XTHEADBB-NEXT: slli a2, a1, 32
+; RV64XTHEADBB-NEXT: add a1, a1, a2
+; RV64XTHEADBB-NEXT: and a0, a0, a1
+; RV64XTHEADBB-NEXT: slli a1, a0, 8
+; RV64XTHEADBB-NEXT: add a0, a0, a1
+; RV64XTHEADBB-NEXT: slli a1, a0, 16
+; RV64XTHEADBB-NEXT: add a0, a0, a1
+; RV64XTHEADBB-NEXT: slli a1, a0, 32
+; RV64XTHEADBB-NEXT: add a0, a0, a1
+; RV64XTHEADBB-NEXT: srli a0, a0, 56
+; RV64XTHEADBB-NEXT: ret
%1 = call i64 @llvm.ctpop.i64(i64 %a)
ret i64 %1
}
diff --git a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
index d57c4d653b2ae..fe6e20d852d59 100644
--- a/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
+++ b/llvm/test/CodeGen/RISCV/ctz_zero_return_test.ll
@@ -602,11 +602,14 @@ define signext i32 @ctlz(i64 %b) nounwind {
;
; RV32I-LABEL: ctlz:
; RV32I: # %bb.0: # %entry
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a4, a2, 1365
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a3, a2, 819
+; RV32I-NEXT: lui a2, 61681
+; RV32I-NEXT: addi a2, a2, -241
+; RV32I-NEXT: bnez a1, .LBB7_2
+; RV32I-NEXT: # %bb.1: # %entry
; RV32I-NEXT: srli a1, a0, 1
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 2
@@ -618,10 +621,27 @@ define signext i32 @ctlz(i64 %b) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: srli a0, s1, 1
-; RV32I-NEXT: or a0, s1, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: addi a0, a0, 32
+; RV32I-NEXT: andi a0, a0, 63
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB7_2:
+; RV32I-NEXT: srli a0, a1, 1
+; RV32I-NEXT: or a0, a1, a0
; RV32I-NEXT: srli a1, a0, 2
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 4
@@ -631,22 +651,26 @@ define signext i32 @ctlz(i64 %b) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: bnez s1, .LBB7_2
-; RV32I-NEXT: # %bb.1: # %entry
-; RV32I-NEXT: addi a0, s0, 32
-; RV32I-NEXT: .LBB7_2: # %entry
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: andi a0, a0, 63
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: ret
;
; RV64I-LABEL: ctlz:
; RV64I: # %bb.0: # %entry
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: srli a1, a0, 1
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: srli a1, a0, 2
@@ -660,10 +684,36 @@ define signext i32 @ctlz(i64 %b) nounwind {
; RV64I-NEXT: srli a1, a0, 32
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: not a0, a0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: andi a0, a0, 63
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a0, a0, 2
+; RV64I-NEXT: srli a0, a0, 58
; RV64I-NEXT: ret
diff --git a/llvm/test/CodeGen/RISCV/pr56457.ll b/llvm/test/CodeGen/RISCV/pr56457.ll
index 19cc8b3af208f..ba08aa838bf99 100644
--- a/llvm/test/CodeGen/RISCV/pr56457.ll
+++ b/llvm/test/CodeGen/RISCV/pr56457.ll
@@ -9,8 +9,6 @@ define i15 @foo(i15 %x) nounwind {
; CHECK-NEXT: slli a1, a0, 49
; CHECK-NEXT: beqz a1, .LBB0_2
; CHECK-NEXT: # %bb.1: # %cond.false
-; CHECK-NEXT: addi sp, sp, -16
-; CHECK-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; CHECK-NEXT: srli a1, a1, 50
; CHECK-NEXT: or a0, a0, a1
; CHECK-NEXT: slli a1, a0, 49
@@ -23,11 +21,34 @@ define i15 @foo(i15 %x) nounwind {
; CHECK-NEXT: srli a1, a1, 57
; CHECK-NEXT: or a0, a0, a1
; CHECK-NEXT: not a0, a0
+; CHECK-NEXT: srli a1, a0, 1
+; CHECK-NEXT: lui a2, 1
+; CHECK-NEXT: addiw a2, a2, 1365
+; CHECK-NEXT: and a1, a1, a2
; CHECK-NEXT: slli a0, a0, 49
; CHECK-NEXT: srli a0, a0, 49
-; CHECK-NEXT: call __popcountdi2
-; CHECK-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT: addi sp, sp, 16
+; CHECK-NEXT: sub a0, a0, a1
+; CHECK-NEXT: lui a1, 209715
+; CHECK-NEXT: addiw a1, a1, 819
+; CHECK-NEXT: slli a2, a1, 32
+; CHECK-NEXT: add a1, a1, a2
+; CHECK-NEXT: and a2, a0, a1
+; CHECK-NEXT: srli a0, a0, 2
+; CHECK-NEXT: and a0, a0, a1
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: srli a1, a0, 4
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: lui a1, 61681
+; CHECK-NEXT: addiw a1, a1, -241
+; CHECK-NEXT: slli a2, a1, 32
+; CHECK-NEXT: add a1, a1, a2
+; CHECK-NEXT: and a0, a0, a1
+; CHECK-NEXT: lui a1, 4112
+; CHECK-NEXT: addiw a1, a1, 257
+; CHECK-NEXT: slli a2, a1, 32
+; CHECK-NEXT: add a1, a1, a2
+; CHECK-NEXT: mul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 56
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB0_2:
; CHECK-NEXT: li a0, 15
diff --git a/llvm/test/CodeGen/RISCV/pr95271.ll b/llvm/test/CodeGen/RISCV/pr95271.ll
index e9a5d2d8de750..950e6fb5f37ec 100644
--- a/llvm/test/CodeGen/RISCV/pr95271.ll
+++ b/llvm/test/CodeGen/RISCV/pr95271.ll
@@ -7,7 +7,28 @@ define i32 @PR95271(ptr %p) {
; RV32I: # %bb.0:
; RV32I-NEXT: lw a0, 0(a0)
; RV32I-NEXT: addi a0, a0, 1
-; RV32I-NEXT: tail __popcountsi2
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: ret
;
; RV64I-LABEL: PR95271:
; RV64I: # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
index ffebc73a34690..197366e7e05fe 100644
--- a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
@@ -11,8 +11,6 @@ define i32 @ctlz_i32(i32 %a) nounwind {
; RV32I: # %bb.0:
; RV32I-NEXT: beqz a0, .LBB0_2
; RV32I-NEXT: # %bb.1: # %cond.false
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32I-NEXT: srli a1, a0, 1
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 2
@@ -24,9 +22,27 @@ define i32 @ctlz_i32(i32 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: ret
; RV32I-NEXT: .LBB0_2:
; RV32I-NEXT: li a0, 32
@@ -45,11 +61,14 @@ declare i64 @llvm.ctlz.i64(i64, i1)
define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-LABEL: ctlz_i64:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a4, a2, 1365
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a3, a2, 819
+; RV32I-NEXT: lui a2, 61681
+; RV32I-NEXT: addi a2, a2, -241
+; RV32I-NEXT: bnez a1, .LBB1_2
+; RV32I-NEXT: # %bb.1:
; RV32I-NEXT: srli a1, a0, 1
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 2
@@ -61,10 +80,27 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: srli a0, s1, 1
-; RV32I-NEXT: or a0, s1, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: addi a0, a0, 32
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB1_2:
+; RV32I-NEXT: srli a0, a1, 1
+; RV32I-NEXT: or a0, a1, a0
; RV32I-NEXT: srli a1, a0, 2
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 4
@@ -74,16 +110,22 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: bnez s1, .LBB1_2
-; RV32I-NEXT: # %bb.1:
-; RV32I-NEXT: addi a0, s0, 32
-; RV32I-NEXT: .LBB1_2:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: li a1, 0
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: ret
;
; RV32XTHEADBB-LABEL: ctlz_i64:
diff --git a/llvm/test/CodeGen/RISCV/rv32zbb.ll b/llvm/test/CodeGen/RISCV/rv32zbb.ll
index 86e0d6b7b3f9d..fa320f53cec6c 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbb.ll
@@ -11,8 +11,6 @@ define i32 @ctlz_i32(i32 %a) nounwind {
; RV32I: # %bb.0:
; RV32I-NEXT: beqz a0, .LBB0_2
; RV32I-NEXT: # %bb.1: # %cond.false
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32I-NEXT: srli a1, a0, 1
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 2
@@ -24,9 +22,27 @@ define i32 @ctlz_i32(i32 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: ret
; RV32I-NEXT: .LBB0_2:
; RV32I-NEXT: li a0, 32
@@ -45,11 +61,14 @@ declare i64 @llvm.ctlz.i64(i64, i1)
define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-LABEL: ctlz_i64:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a4, a2, 1365
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a3, a2, 819
+; RV32I-NEXT: lui a2, 61681
+; RV32I-NEXT: addi a2, a2, -241
+; RV32I-NEXT: bnez a1, .LBB1_2
+; RV32I-NEXT: # %bb.1:
; RV32I-NEXT: srli a1, a0, 1
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 2
@@ -61,10 +80,27 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: srli a0, s1, 1
-; RV32I-NEXT: or a0, s1, a0
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: addi a0, a0, 32
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: ret
+; RV32I-NEXT: .LBB1_2:
+; RV32I-NEXT: srli a0, a1, 1
+; RV32I-NEXT: or a0, a1, a0
; RV32I-NEXT: srli a1, a0, 2
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: srli a1, a0, 4
@@ -74,16 +110,22 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV32I-NEXT: srli a1, a0, 16
; RV32I-NEXT: or a0, a0, a1
; RV32I-NEXT: not a0, a0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: bnez s1, .LBB1_2
-; RV32I-NEXT: # %bb.1:
-; RV32I-NEXT: addi a0, s0, 32
-; RV32I-NEXT: .LBB1_2:
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: and a1, a0, a3
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
; RV32I-NEXT: li a1, 0
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: ret
;
; RV32ZBB-LABEL: ctlz_i64:
@@ -211,7 +253,28 @@ declare i32 @llvm.ctpop.i32(i32)
define i32 @ctpop_i32(i32 %a) nounwind {
; RV32I-LABEL: ctpop_i32:
; RV32I: # %bb.0:
-; RV32I-NEXT: tail __popcountsi2
+; RV32I-NEXT: srli a1, a0, 1
+; RV32I-NEXT: lui a2, 349525
+; RV32I-NEXT: addi a2, a2, 1365
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: sub a0, a0, a1
+; RV32I-NEXT: lui a1, 209715
+; RV32I-NEXT: addi a1, a1, 819
+; RV32I-NEXT: and a2, a0, a1
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: add a0, a2, a0
+; RV32I-NEXT: srli a1, a0, 4
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lui a1, 61681
+; RV32I-NEXT: addi a1, a1, -241
+; RV32I-NEXT: and a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: slli a1, a0, 16
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: ret
;
; RV32ZBB-LABEL: ctpop_i32:
; RV32ZBB: # %bb.0:
@@ -302,21 +365,42 @@ declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
define <2 x i32> @ctpop_v2i32(<2 x i32> %a) nounwind {
; RV32I-LABEL: ctpop_v2i32:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT: mv s0, a1
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: mv a0, s0
-; RV32I-NEXT: call __popcountsi2
-; RV32I-NEXT: mv a1, a0
-; RV32I-NEXT: mv a0, s1
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: srli a2, a0, 1
+; RV32I-NEXT: lui a3, 349525
+; RV32I-NEXT: addi a3, a3, 1365
+; RV32I-NEXT: and a2, a2, a3
+; RV32I-NEXT: sub a0, a0, a2
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a2, a2, 819
+; RV32I-NEXT: and a4, a0, a2
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: add a0, a4, a0
+; RV32I-NEXT: srli a4, a0, 4
+; RV32I-NEXT: add a0, a0, a4
+; RV32I-NEXT: lui a4, 61681
+; RV32I-NEXT: addi a4, a4, -241
+; RV32I-NEXT: and a0, a0, a4
+; RV32I-NEXT: slli a5, a0, 8
+; RV32I-NEXT: add a0, a0, a5
+; RV32I-NEXT: slli a5, a0, 16
+; RV32I-NEXT: add a0, a0, a5
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: srli a5, a1, 1
+; RV32I-NEXT: and a3, a5, a3
+; RV32I-NEXT: sub a1, a1, a3
+; RV32I-NEXT: and a3, a1, a2
+; RV32I-NEXT: srli a1, a1, 2
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: add a1, a3, a1
+; RV32I-NEXT: srli a2, a1, 4
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: slli a2, a1, 8
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: slli a2, a1, 16
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: srli a1, a1, 24
; RV32I-NEXT: ret
;
; RV32ZBB-LABEL: ctpop_v2i32:
@@ -433,11 +517,44 @@ declare i64 @llvm.ctpop.i64(i64)
define i64 @ctpop_i64(i64 %a) nounwind {
; RV32I-LABEL: ctpop_i64:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -16
-; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: call __popcountdi2
-; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: srli a2, a1, 1
+; RV32I-NEXT: lui a3, 349525
+; RV32I-NEXT: addi a3, a3, 1365
+; RV32I-NEXT: and a2, a2, a3
+; RV32I-NEXT: sub a1, a1, a2
+; RV32I-NEXT: lui a2, 209715
+; RV32I-NEXT: addi a2, a2, 819
+; RV32I-NEXT: and a4, a1, a2
+; RV32I-NEXT: srli a1, a1, 2
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: add a1, a4, a1
+; RV32I-NEXT: srli a4, a1, 4
+; RV32I-NEXT: add a1, a1, a4
+; RV32I-NEXT: lui a4, 61681
+; RV32I-NEXT: addi a4, a4, -241
+; RV32I-NEXT: and a1, a1, a4
+; RV32I-NEXT: slli a5, a1, 8
+; RV32I-NEXT: add a1, a1, a5
+; RV32I-NEXT: slli a5, a1, 16
+; RV32I-NEXT: add a1, a1, a5
+; RV32I-NEXT: srli a1, a1, 24
+; RV32I-NEXT: srli a5, a0, 1
+; RV32I-NEXT: and a3, a5, a3
+; RV32I-NEXT: sub a0, a0, a3
+; RV32I-NEXT: and a3, a0, a2
+; RV32I-NEXT: srli a0, a0, 2
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: add a0, a3, a0
+; RV32I-NEXT: srli a2, a0, 4
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: and a0, a0, a4
+; RV32I-NEXT: slli a2, a0, 8
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: slli a2, a0, 16
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: srli a0, a0, 24
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: li a1, 0
; RV32I-NEXT: ret
;
; RV32ZBB-LABEL: ctpop_i64:
@@ -565,36 +682,82 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
; RV32I-LABEL: ctpop_v2i64:
; RV32I: # %bb.0:
-; RV32I-NEXT: addi sp, sp, -32
-; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT: lw s0, 8(a1)
-; RV32I-NEXT: lw s1, 12(a1)
-; RV32I-NEXT: lw a2, 0(a1)
-; RV32I-NEXT: lw a1, 4(a1)
-; RV32I-NEXT: mv s2, a0
-; RV32I-NEXT: mv a0, a2
-; RV32I-NEXT: call __popcountdi2
-; RV32I-NEXT: mv s3, a0
-; RV32I-NEXT: mv s4, a1
-; RV32I-NEXT: mv a0, s0
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call __popcountdi2
-; RV32I-NEXT: sw a1, 12(s2)
-; RV32I-NEXT: sw a0, 8(s2)
-; RV32I-NEXT: sw s4, 4(s2)
-; RV32I-NEXT: sw s3, 0(s2)
-; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: lw a3, 4(a1)
+; RV32I-NEXT: lw a2, 8(a1)
+; RV32I-NEXT: lw a4, 12(a1)
+; RV32I-NEXT: lw a1, 0(a1)
+; RV32I-NEXT: srli a5, a3, 1
+; RV32I-NEXT: lui a6, 349525
+; RV32I-NEXT: addi a6, a6, 1365
+; RV32I-NEXT: and a5, a5, a6
+; RV32I-NEXT: sub a3, a3, a5
+; RV32I-NEXT: lui a5, 209715
+; RV32I-NEXT: addi a5, a5, 819
+; RV32I-NEXT: and a7, a3, a5
+; RV32I-NEXT: srli a3, a3, 2
+; RV32I-NEXT: and a3, a3, a5
+; RV32I-NEXT: add a3, a7, a3
+; RV32I-NEXT: srli a7, a3, 4
+; RV32I-NEXT: add a3, a3, a7
+; RV32I-NEXT: lui a7, 61681
+; RV32I-NEXT: addi a7, a7, -241
+; RV32I-NEXT: and a3, a3, a7
+; RV32I-NEXT: slli t0, a3, 8
+; RV32I-NEXT: add a3, a3, t0
+; RV32I-NEXT: slli t0, a3, 16
+; RV32I-NEXT: add a3, a3, t0
+; RV32I-NEXT: srli a3, a3, 24
+; RV32I-NEXT: srli t0, a1, 1
+; RV32I-NEXT: and t0, t0, a6
+; RV32I-NEXT: sub a1, a1, t0
+; RV32I-NEXT: and t0, a1, a5
+; RV32I-NEXT: srli a1, a1, 2
+; RV32I-NEXT: and a1, a1, a5
+; RV32I-NEXT: add a1, t0, a1
+; RV32I-NEXT: srli t0, a1, 4
+; RV32I-NEXT: add a1, a1, t0
+; RV32I-NEXT: and a1, a1, a7
+; RV32I-NEXT: slli t0, a1, 8
+; RV32I-NEXT: add a1, a1, t0
+; RV32I-NEXT: slli t0, a1, 16
+; RV32I-NEXT: add a1, a1, t0
+; RV32I-NEXT: srli a1, a1, 24
+; RV32I-NEXT: add a1, a1, a3
+; RV32I-NEXT: srli a3, a4, 1
+; RV32I-NEXT: and a3, a3, a6
+; RV32I-NEXT: sub a4, a4, a3
+; RV32I-NEXT: and a3, a4, a5
+; RV32I-NEXT: srli a4, a4, 2
+; RV32I-NEXT: and a4, a4, a5
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: srli a4, a3, 4
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: and a3, a3, a7
+; RV32I-NEXT: slli a4, a3, 8
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: slli a4, a3, 16
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: srli a3, a3, 24
+; RV32I-NEXT: srli a4, a2, 1
+; RV32I-NEXT: and a4, a4, a6
+; RV32I-NEXT: sub a2, a2, a4
+; RV32I-NEXT: and a4, a2, a5
+; RV32I-NEXT: srli a2, a2, 2
+; RV32I-NEXT: and a2, a2, a5
+; RV32I-NEXT: add a2, a4, a2
+; RV32I-NEXT: srli a4, a2, 4
+; RV32I-NEXT: add a2, a2, a4
+; RV32I-NEXT: and a2, a2, a7
+; RV32I-NEXT: slli a4, a2, 8
+; RV32I-NEXT: add a2, a2, a4
+; RV32I-NEXT: slli a4, a2, 16
+; RV32I-NEXT: add a2, a2, a4
+; RV32I-NEXT: srli a2, a2, 24
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sw zero, 12(a0)
+; RV32I-NEXT: sw zero, 4(a0)
+; RV32I-NEXT: sw a2, 8(a0)
+; RV32I-NEXT: sw a1, 0(a0)
; RV32I-NEXT: ret
;
; RV32ZBB-LABEL: ctpop_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
index eb8b5c9dd695f..80d3add385969 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll
@@ -317,8 +317,6 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I: # %bb.0:
; RV64I-NEXT: beqz a0, .LBB5_2
; RV64I-NEXT: # %bb.1: # %cond.false
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: srli a1, a0, 1
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: srli a1, a0, 2
@@ -332,9 +330,35 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I-NEXT: srli a1, a0, 32
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: not a0, a0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: ret
; RV64I-NEXT: .LBB5_2:
; RV64I-NEXT: li a0, 64
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
index 445b4dc671296..b0e447b71178b 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll
@@ -307,8 +307,6 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I: # %bb.0:
; RV64I-NEXT: beqz a0, .LBB5_2
; RV64I-NEXT: # %bb.1: # %cond.false
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: srli a1, a0, 1
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: srli a1, a0, 2
@@ -322,9 +320,35 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I-NEXT: srli a1, a0, 32
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: not a0, a0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: ret
; RV64I-NEXT: .LBB5_2:
; RV64I-NEXT: li a0, 64
@@ -596,7 +620,36 @@ declare i64 @llvm.ctpop.i64(i64)
define i64 @ctpop_i64(i64 %a) nounwind {
; RV64I-LABEL: ctpop_i64:
; RV64I: # %bb.0:
-; RV64I-NEXT: tail __popcountdi2
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: ret
;
; RV64ZBB-LABEL: ctpop_i64:
; RV64ZBB: # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index c667b930c1ee0..6cdab888ffcde 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -295,8 +295,6 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I: # %bb.0:
; RV64I-NEXT: beqz a0, .LBB5_2
; RV64I-NEXT: # %bb.1: # %cond.false
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: srli a1, a0, 1
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: srli a1, a0, 2
@@ -310,9 +308,35 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I-NEXT: srli a1, a0, 32
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: not a0, a0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: ret
; RV64I-NEXT: .LBB5_2:
; RV64I-NEXT: li a0, 64
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index 53d7f77285e72..43a499806ab5a 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -285,8 +285,6 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I: # %bb.0:
; RV64I-NEXT: beqz a0, .LBB5_2
; RV64I-NEXT: # %bb.1: # %cond.false
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: srli a1, a0, 1
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: srli a1, a0, 2
@@ -300,9 +298,35 @@ define i64 @ctlz_i64(i64 %a) nounwind {
; RV64I-NEXT: srli a1, a0, 32
; RV64I-NEXT: or a0, a0, a1
; RV64I-NEXT: not a0, a0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: ret
; RV64I-NEXT: .LBB5_2:
; RV64I-NEXT: li a0, 64
@@ -804,7 +828,36 @@ declare i64 @llvm.ctpop.i64(i64)
define i64 @ctpop_i64(i64 %a) nounwind {
; RV64I-LABEL: ctpop_i64:
; RV64I: # %bb.0:
-; RV64I-NEXT: tail __popcountdi2
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: lui a2, 349525
+; RV64I-NEXT: addiw a2, a2, 1365
+; RV64I-NEXT: slli a3, a2, 32
+; RV64I-NEXT: add a2, a2, a3
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw a1, a1, 819
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a2, a0, a1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: add a0, a2, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw a1, a1, -241
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 8
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 16
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: slli a1, a0, 32
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: ret
;
; RV64ZBB-LABEL: ctpop_i64:
; RV64ZBB: # %bb.0:
@@ -895,21 +948,52 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
define <2 x i64> @ctpop_v2i64(<2 x i64> %a) nounwind {
; RV64I-LABEL: ctpop_v2i64:
; RV64I: # %bb.0:
-; RV64I-NEXT: addi sp, sp, -32
-; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT: mv s0, a1
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: mv s1, a0
-; RV64I-NEXT: mv a0, s0
-; RV64I-NEXT: call __popcountdi2
-; RV64I-NEXT: mv a1, a0
-; RV64I-NEXT: mv a0, s1
-; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
-; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: srli a2, a0, 1
+; RV64I-NEXT: lui a3, 349525
+; RV64I-NEXT: addiw a3, a3, 1365
+; RV64I-NEXT: slli a4, a3, 32
+; RV64I-NEXT: add a3, a3, a4
+; RV64I-NEXT: and a2, a2, a3
+; RV64I-NEXT: sub a0, a0, a2
+; RV64I-NEXT: lui a2, 209715
+; RV64I-NEXT: addiw a2, a2, 819
+; RV64I-NEXT: slli a4, a2, 32
+; RV64I-NEXT: add a2, a2, a4
+; RV64I-NEXT: and a4, a0, a2
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, a2
+; RV64I-NEXT: add a0, a4, a0
+; RV64I-NEXT: srli a4, a0, 4
+; RV64I-NEXT: add a0, a0, a4
+; RV64I-NEXT: lui a4, 61681
+; RV64I-NEXT: addiw a4, a4, -241
+; RV64I-NEXT: slli a5, a4, 32
+; RV64I-NEXT: add a4, a4, a5
+; RV64I-NEXT: and a0, a0, a4
+; RV64I-NEXT: slli a5, a0, 8
+; RV64I-NEXT: add a0, a0, a5
+; RV64I-NEXT: slli a5, a0, 16
+; RV64I-NEXT: add a0, a0, a5
+; RV64I-NEXT: slli a5, a0, 32
+; RV64I-NEXT: add a0, a0, a5
+; RV64I-NEXT: srli a0, a0, 56
+; RV64I-NEXT: srli a5, a1, 1
+; RV64I-NEXT: and a3, a5, a3
+; RV64I-NEXT: sub a1, a1, a3
+; RV64I-NEXT: and a3, a1, a2
+; RV64I-NEXT: srli a1, a1, 2
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: add a1, a3, a1
+; RV64I-NEXT: srli a2, a1, 4
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: and a1, a1, a4
+; RV64I-NEXT: slli a2, a1, 8
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: slli a2, a1, 16
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: slli a2, a1, 32
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: srli a1, a1, 56
; RV64I-NEXT: ret
;
; RV64ZBB-LABEL: ctpop_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/sextw-removal.ll b/llvm/test/CodeGen/RISCV/sextw-removal.ll
index f9a2c543d5410..8cf78551d28f9 100644
--- a/llvm/test/CodeGen/RISCV/sextw-removal.ll
+++ b/llvm/test/CodeGen/RISCV/sextw-removal.ll
@@ -316,18 +316,52 @@ declare float @baz(i32 signext %i3)
define void @test7(i32 signext %arg, i32 signext %arg1) nounwind {
; RV64I-LABEL: test7:
; RV64I: # %bb.0: # %bb
-; RV64I-NEXT: addi sp, sp, -16
-; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: sraw a0, a0, a1
+; RV64I-NEXT: lui a1, 349525
+; RV64I-NEXT: addiw s0, a1, 1365
+; RV64I-NEXT: slli a1, s0, 32
+; RV64I-NEXT: add s0, s0, a1
+; RV64I-NEXT: lui a1, 209715
+; RV64I-NEXT: addiw s1, a1, 819
+; RV64I-NEXT: slli a1, s1, 32
+; RV64I-NEXT: add s1, s1, a1
+; RV64I-NEXT: lui a1, 61681
+; RV64I-NEXT: addiw s2, a1, -241
+; RV64I-NEXT: slli a1, s2, 32
+; RV64I-NEXT: add s2, s2, a1
+; RV64I-NEXT: lui a1, 4112
+; RV64I-NEXT: addiw s3, a1, 257
+; RV64I-NEXT: slli a1, s3, 32
+; RV64I-NEXT: add s3, s3, a1
; RV64I-NEXT: .LBB6_1: # %bb2
; RV64I-NEXT: # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT: sext.w a0, a0
; RV64I-NEXT: call foo
-; RV64I-NEXT: call __popcountdi2
+; RV64I-NEXT: srli a1, a0, 1
+; RV64I-NEXT: and a1, a1, s0
+; RV64I-NEXT: sub a0, a0, a1
+; RV64I-NEXT: and a1, a0, s1
+; RV64I-NEXT: srli a0, a0, 2
+; RV64I-NEXT: and a0, a0, s1
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: srli a1, a0, 4
+; RV64I-NEXT: add a0, a0, a1
+; RV64I-NEXT: and a0, a0, s2
+; RV64I-NEXT: mul a0, a0, s3
+; RV64I-NEXT: srli a0, a0, 56
; RV64I-NEXT: bnez a0, .LBB6_1
; RV64I-NEXT: # %bb.2: # %bb7
-; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
-; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
; RV64I-NEXT: ret
;
; RV64ZBB-LABEL: test7:
More information about the llvm-branch-commits
mailing list