[llvm] 4510d55 - [LLVM] Add llvm.[su]mulh intrinsics. (#220293)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 04:58:29 PDT 2026
Author: Ricardo Jesus
Date: 2026-09-15T11:58:22Z
New Revision: 4510d555b04130df9fee7af75c34955d1b0d156a
URL: https://github.com/llvm/llvm-project/commit/4510d555b04130df9fee7af75c34955d1b0d156a
DIFF: https://github.com/llvm/llvm-project/commit/4510d555b04130df9fee7af75c34955d1b0d156a.diff
LOG: [LLVM] Add llvm.[su]mulh intrinsics. (#220293)
These intrinsics correspond to (un)signed multiply-high operations. The
motivation for introducing them is to allow more accurate costing
(particularly for vectorisation) and to simplify code generation for
chained multiply-high operations.
Together with subsequent patches to combine and cost the intrinsics,
this gives an 8% speedup on 750.sealcrypto_r on Neoverse V2.
Added:
llvm/test/CodeGen/AArch64/smulh.ll
llvm/test/CodeGen/AArch64/sve-smulh.ll
llvm/test/CodeGen/AArch64/sve-umulh.ll
llvm/test/CodeGen/AArch64/umulh.ll
llvm/test/CodeGen/RISCV/smulh.ll
llvm/test/CodeGen/RISCV/umulh.ll
llvm/test/CodeGen/X86/smulh.ll
llvm/test/CodeGen/X86/umulh.ll
Modified:
llvm/docs/LangRef.md
llvm/docs/ReleaseNotes.md
llvm/include/llvm/CodeGen/TargetLowering.h
llvm/include/llvm/IR/IntrinsicInst.h
llvm/include/llvm/IR/Intrinsics.td
llvm/lib/Analysis/ConstantFolding.cpp
llvm/lib/Analysis/ValueTracking.cpp
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
Removed:
################################################################################
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index a29e8f7508a7f..f9ac1d6872c9a 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -19900,6 +19900,84 @@ This never sets errno, just as '`llvm.fma.*`'.
%r2 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) ; yields float:r2 = (a * b) + c
```
+(int_smulh)=
+
+#### '`llvm.smulh.*`' Intrinsic
+
+##### Syntax:
+
+This is an overloaded intrinsic. You can use `llvm.smulh` on any integer
+or vector of integers.
+
+```
+declare i16 @llvm.smulh.i16(i16 %a, i16 %b)
+declare i32 @llvm.smulh.i32(i32 %a, i32 %b)
+declare i64 @llvm.smulh.i64(i64 %a, i64 %b)
+declare <4 x i32> @llvm.smulh.v4i32(<4 x i32> %a, <4 x i32> %b)
+```
+
+##### Overview:
+
+The '`llvm.smulh`' family of intrinsic functions performs a signed
+multiplication of the two arguments, and returns the high half of the result.
+
+##### Arguments:
+
+The arguments may be any integer type or vector of integer type. Both
+arguments and result must have the same type.
+
+##### Semantics:
+
+The '`llvm.smulh`' intrinsic computes signed multiply-high of its arguments,
+which is the upper N-bit half of the 2N-bit product for signed iN types.
+
+##### Example:
+
+```llvm
+%r = call i4 @llvm.smulh.i4(i4 1, i4 2) ; %r = 0
+%r = call i4 @llvm.smulh.i4(i4 5, i4 6) ; %r = 1
+%r = call i4 @llvm.smulh.i4(i4 -4, i4 6) ; %r = -2
+```
+
+(int_umulh)=
+
+#### '`llvm.umulh.*`' Intrinsic
+
+##### Syntax:
+
+This is an overloaded intrinsic. You can use `llvm.umulh` on any integer
+or vector of integers.
+
+```
+declare i16 @llvm.umulh.i16(i16 %a, i16 %b)
+declare i32 @llvm.umulh.i32(i32 %a, i32 %b)
+declare i64 @llvm.umulh.i64(i64 %a, i64 %b)
+declare <4 x i32> @llvm.umulh.v4i32(<4 x i32> %a, <4 x i32> %b)
+```
+
+##### Overview:
+
+The '`llvm.umulh`' family of intrinsic functions performs an unsigned
+multiplication of the two arguments, and returns the high half of the result.
+
+##### Arguments:
+
+The arguments may be any integer type or vector of integer type. Both
+arguments and result must have the same type.
+
+##### Semantics:
+
+The '`llvm.umulh`' intrinsic computes unsigned multiply-high of its arguments,
+which is the upper N-bit half of the 2N-bit product for unsigned iN types.
+
+##### Example:
+
+```llvm
+%r = call i4 @llvm.umulh.i4(i4 1, i4 2) ; %r = 0
+%r = call i4 @llvm.umulh.i4(i4 5, i4 6) ; %r = 1
+%r = call i4 @llvm.umulh.i4(i4 4, i4 10) ; %r = 2
+```
+
### Hardware-Loop Intrinsics
LLVM support several intrinsics to mark a loop as a hardware-loop. They are
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 4acfcaf3204a7..9951c415b982e 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -75,6 +75,8 @@ Makes programs 10x faster by doing Special New Thing.
* Added `llvm.vector.reduce.fmaximumnum` and `llvm.vector.reduce.fminimumnum`
intrinsics, the reduction variants of `llvm.maximumnum` and
`llvm.minimumnum`.
+* Added `llvm.smulh` and `llvm.umulh` intrinsics for signed and unsigned
+ multiply returning the high-order half of the 2N-bit product of iN operands.
* Added `nofreeobj` attribute for attributes and returns, which forbids
freeing the underlying object (as opposed to only frees through that specific
pointer). Renamed `!nofree` metadata to `!nofreeobj`, as it has the same
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 4006ad3bda5d6..cd6ec343a8bba 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5995,6 +5995,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
SelectionDAG &DAG) const;
+ // Expand ISD::MULH[SU]. Can expand to MUL_LOHI or wide MUL if available.
+ SDValue expandMULH(SDNode *Node, SelectionDAG &DAG) const;
+
/// Calculate the product twice the width of LHS and RHS. If HiLHS/HiRHS are
/// non-null they will be included in the multiplication. The expansion works
/// by splitting the 2 inputs into 4 pieces that we can multiply and add
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 7c0d9889a6d1a..7d0bb92114e84 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -93,6 +93,8 @@ class IntrinsicInst : public CallInst {
case Intrinsic::umul_fix:
case Intrinsic::smul_fix_sat:
case Intrinsic::umul_fix_sat:
+ case Intrinsic::smulh:
+ case Intrinsic::umulh:
case Intrinsic::fma:
case Intrinsic::fmuladd:
return true;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index a376d4aa12af1..ff53373ca502d 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1645,6 +1645,10 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrNoCreateUndefOrPoison]
[LLVMMatchType<0>, LLVMMatchType<0>]>;
def int_pdep : DefaultAttrsIntrinsic<[llvm_anyint_ty],
[LLVMMatchType<0>, LLVMMatchType<0>]>;
+ def int_smulh : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>]>;
+ def int_umulh : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>]>;
}
let IntrProperties = [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<1>>] in {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index aca2556cd48eb..945df4f20f8a0 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1760,6 +1760,8 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
case Intrinsic::usub_with_overflow:
case Intrinsic::smul_with_overflow:
case Intrinsic::umul_with_overflow:
+ case Intrinsic::smulh:
+ case Intrinsic::umulh:
case Intrinsic::sadd_sat:
case Intrinsic::uadd_sat:
case Intrinsic::ssub_sat:
@@ -3898,6 +3900,14 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
if (!C0 || !C1)
return Constant::getNullValue(Ty);
return ConstantInt::get(Ty, APIntOps::pext(*C0, *C1));
+ case Intrinsic::smulh:
+ if (!C0 || !C1)
+ return Constant::getNullValue(Ty);
+ return ConstantInt::get(Ty, APIntOps::mulhs(*C0, *C1));
+ case Intrinsic::umulh:
+ if (!C0 || !C1)
+ return Constant::getNullValue(Ty);
+ return ConstantInt::get(Ty, APIntOps::mulhu(*C0, *C1));
case Intrinsic::amdgcn_wave_reduce_umin:
case Intrinsic::amdgcn_wave_reduce_umax:
case Intrinsic::amdgcn_wave_reduce_max:
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index ac49dbc8d76d3..e6ad7018cd261 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -2157,6 +2157,16 @@ static void computeKnownBitsFromOperator(const Operator *I,
computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
Known = KnownBits::pdep(Known, Known2);
break;
+ case Intrinsic::smulh:
+ computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+ computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
+ Known = KnownBits::mulhs(Known, Known2);
+ break;
+ case Intrinsic::umulh:
+ computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
+ computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
+ Known = KnownBits::mulhu(Known, Known2);
+ break;
case Intrinsic::uadd_sat:
computeKnownBits(I->getOperand(0), DemandedElts, Known, Q, Depth + 1);
computeKnownBits(I->getOperand(1), DemandedElts, Known2, Q, Depth + 1);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index f4d46e3cad48d..05d2b4815dcf0 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2740,6 +2740,10 @@ unsigned IRTranslatorImpl::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_FCOSH;
case Intrinsic::ctpop:
return TargetOpcode::G_CTPOP;
+ case Intrinsic::smulh:
+ return TargetOpcode::G_SMULH;
+ case Intrinsic::umulh:
+ return TargetOpcode::G_UMULH;
case Intrinsic::exp:
return TargetOpcode::G_FEXP;
case Intrinsic::exp2:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 29f1335563166..bd9c47806a9ca 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4109,8 +4109,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
- unsigned MULHOpcode =
- Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
+ bool IsSigned = Node->getOpcode() == ISD::SMUL_LOHI;
+ unsigned MULHOpcode = IsSigned ? ISD::MULHS : ISD::MULHU;
if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
@@ -4120,8 +4120,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
SmallVector<SDValue, 4> Halves;
EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
- assert(TLI.isTypeLegal(HalfType));
- if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
+ if (TLI.isTypeLegal(HalfType) &&
+ TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
HalfType, DAG,
TargetLowering::MulExpansionKind::Always)) {
for (unsigned i = 0; i < 2; ++i) {
@@ -4134,6 +4134,11 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
break;
}
+
+ SDValue Lo, Hi;
+ TLI.forceExpandWideMUL(DAG, dl, IsSigned, LHS, RHS, Lo, Hi);
+ Results.push_back(Lo);
+ Results.push_back(Hi);
break;
}
case ISD::MUL: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 43e41ed39ca1d..7d61c3445ab9f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -347,6 +347,11 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_PDEP(N);
break;
+ case ISD::MULHS:
+ case ISD::MULHU:
+ Res = PromoteIntRes_MULH(N);
+ break;
+
case ISD::IS_FPCLASS:
Res = PromoteIntRes_IS_FPCLASS(N);
break;
@@ -1694,6 +1699,35 @@ SDValue DAGTypeLegalizer::PromoteIntRes_PDEP(SDNode *N) {
return DAG.getNode(ISD::PDEP, DL, VT, X, Y);
}
+SDValue DAGTypeLegalizer::PromoteIntRes_MULH(SDNode *N) {
+ SDLoc dl(N);
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ bool IsSigned = N->getOpcode() == ISD::MULHS;
+ unsigned BW = VT.getScalarSizeInBits();
+ unsigned NBW = NVT.getScalarSizeInBits();
+
+ SDValue LHS, RHS;
+ if (IsSigned) {
+ LHS = SExtPromotedInteger(N->getOperand(0));
+ RHS = SExtPromotedInteger(N->getOperand(1));
+ } else {
+ LHS = ZExtPromotedInteger(N->getOperand(0));
+ RHS = ZExtPromotedInteger(N->getOperand(1));
+ }
+
+ // If the promoted type is already wide enough, emit a MUL.
+ if (NBW >= 2 * BW)
+ return DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, NVT,
+ DAG.getNode(ISD::MUL, dl, NVT, LHS, RHS),
+ DAG.getShiftAmountConstant(BW, NVT, dl));
+
+ // Otherwise, align an operand with a left-shift and emit a MULH.
+ LHS = DAG.getNode(ISD::SHL, dl, NVT, LHS,
+ DAG.getShiftAmountConstant(NBW - BW, NVT, dl));
+ return DAG.getNode(N->getOpcode(), dl, NVT, LHS, RHS);
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res;
@@ -3206,6 +3240,11 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
ExpandIntRes_PDEP(N, Lo, Hi);
break;
+ case ISD::MULHS:
+ case ISD::MULHU:
+ ExpandIntRes_MULH(N, Lo, Hi);
+ break;
+
case ISD::VSCALE:
ExpandIntRes_VSCALE(N, Lo, Hi);
break;
@@ -5535,6 +5574,11 @@ void DAGTypeLegalizer::ExpandIntRes_PDEP(SDNode *N, SDValue &Lo, SDValue &Hi) {
SplitInteger(Res, Lo, Hi);
}
+void DAGTypeLegalizer::ExpandIntRes_MULH(SDNode *N, SDValue &Lo, SDValue &Hi) {
+ SDValue Res = TLI.expandMULH(N, DAG);
+ SplitInteger(Res, Lo, Hi);
+}
+
void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
SDValue &Hi) {
EVT VT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 12d1f34beb8ee..2ad7054e74bfa 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -354,6 +354,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_CLMUL(SDNode *N);
SDValue PromoteIntRes_PEXT(SDNode *N);
SDValue PromoteIntRes_PDEP(SDNode *N);
+ SDValue PromoteIntRes_MULH(SDNode *N);
SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
SDValue PromoteIntRes_READ_REGISTER(SDNode *N);
@@ -499,6 +500,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_PEXT(SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_PDEP(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandIntRes_MULH(SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_VSCALE (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 1776c3304b840..5e3f252fdd3d4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1163,6 +1163,13 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::SMULO:
ExpandMULO(Node, Results);
return;
+ case ISD::MULHS:
+ case ISD::MULHU:
+ if (SDValue Expanded = TLI.expandMULH(Node, DAG)) {
+ Results.push_back(Expanded);
+ return;
+ }
+ break;
case ISD::USUBSAT:
case ISD::SSUBSAT:
case ISD::UADDSAT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 794a4e6e1b537..7508e130f6d55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7532,6 +7532,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
setValue(&I, DAG.getNode(ISD::PDEP, sdl, X.getValueType(), X, Y));
return;
}
+ case Intrinsic::smulh:
+ case Intrinsic::umulh: {
+ auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
+ SDValue X = getValue(I.getArgOperand(0));
+ SDValue Y = getValue(I.getArgOperand(1));
+ setValue(&I, DAG.getNode(Opc, sdl, X.getValueType(), X, Y));
+ return;
+ }
case Intrinsic::sadd_sat: {
SDValue Op1 = getValue(I.getArgOperand(0));
SDValue Op2 = getValue(I.getArgOperand(1));
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 993f7a3931a35..6e559d91952f2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13093,6 +13093,41 @@ bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
return true;
}
+SDValue TargetLowering::expandMULH(SDNode *Node, SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ EVT VT = Node->getValueType(0);
+ SDValue LHS = Node->getOperand(0);
+ SDValue RHS = Node->getOperand(1);
+ bool IsSigned = Node->getOpcode() == ISD::MULHS;
+
+ // Use MUL_LOHI if legal/custom for the original type.
+ unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
+ if (isOperationLegalOrCustom(LoHiOp, VT))
+ return DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS).getValue(1);
+
+ // Use a wide multiply if available.
+ EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
+ if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
+ unsigned BW = VT.getScalarSizeInBits();
+ LHS = DAG.getExtOrTrunc(IsSigned, LHS, dl, WideVT);
+ RHS = DAG.getExtOrTrunc(IsSigned, RHS, dl, WideVT);
+ return DAG.getNode(ISD::TRUNCATE, dl, VT,
+ DAG.getNode(ISD::SRL, dl, WideVT,
+ DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS),
+ DAG.getShiftAmountConstant(BW, WideVT, dl)));
+ }
+
+ // Let fixed-length vectors be scalarised by the caller.
+ // Expand everything else with a wide multiply.
+ if (!VT.isFixedLengthVector()) {
+ SDValue Lo, Hi;
+ forceExpandWideMUL(DAG, dl, IsSigned, LHS, RHS, Lo, Hi);
+ return Hi;
+ }
+
+ return SDValue();
+}
+
SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 8548f63bd1150..bf13d22cf05a8 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -1461,6 +1461,26 @@ define i32 @test_fshr_intrinsic(i32 %a, i32 %b, i32 %c) {
ret i32 %res
}
+define i32 @test_smulh_intrinsic(i32 %a, i32 %b) {
+; CHECK-LABEL: name: test_smulh_intrinsic
+; CHECK: [[A:%[0-9]+]]:_(i32) = COPY $w0
+; CHECK: [[B:%[0-9]+]]:_(i32) = COPY $w1
+; CHECK: [[RES:%[0-9]+]]:_(i32) = G_SMULH [[A]], [[B]]
+; CHECK: $w0 = COPY [[RES]]
+ %res = call i32 @llvm.smulh.i32(i32 %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_umulh_intrinsic(i32 %a, i32 %b) {
+; CHECK-LABEL: name: test_umulh_intrinsic
+; CHECK: [[A:%[0-9]+]]:_(i32) = COPY $w0
+; CHECK: [[B:%[0-9]+]]:_(i32) = COPY $w1
+; CHECK: [[RES:%[0-9]+]]:_(i32) = G_UMULH [[A]], [[B]]
+; CHECK: $w0 = COPY [[RES]]
+ %res = call i32 @llvm.umulh.i32(i32 %a, i32 %b)
+ ret i32 %res
+}
+
declare void @llvm.lifetime.start.p0(i64, ptr)
declare void @llvm.lifetime.end.p0(i64, ptr)
define void @test_lifetime_intrin() {
diff --git a/llvm/test/CodeGen/AArch64/smulh.ll b/llvm/test/CodeGen/AArch64/smulh.ll
new file mode 100644
index 0000000000000..1f87558c7c63c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/smulh.ll
@@ -0,0 +1,552 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i16 @smulh_i16(i16 %x, i16 %y) nounwind {
+; CHECK-SD-LABEL: smulh_i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sxth w8, w1
+; CHECK-SD-NEXT: sxth w9, w0
+; CHECK-SD-NEXT: mul w8, w9, w8
+; CHECK-SD-NEXT: lsr w0, w8, #16
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sxth w8, w0
+; CHECK-GI-NEXT: sxth w9, w1
+; CHECK-GI-NEXT: mul w8, w8, w9
+; CHECK-GI-NEXT: asr w0, w8, #16
+; CHECK-GI-NEXT: ret
+ %r = call i16 @llvm.smulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @smulh_i32(i32 %x, i32 %y) nounwind {
+; CHECK-SD-LABEL: smulh_i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull x8, w0, w1
+; CHECK-SD-NEXT: lsr x0, x8, #32
+; CHECK-SD-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull x8, w0, w1
+; CHECK-GI-NEXT: asr x0, x8, #32
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
+ %r = call i32 @llvm.smulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @smulh_i64(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: smulh_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smulh x0, x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.smulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @smulh_i96(i96 %x, i96 %y) nounwind {
+; CHECK-SD-LABEL: smulh_i96:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: extr x9, x1, x0, #32
+; CHECK-SD-NEXT: lsl x10, x0, #32
+; CHECK-SD-NEXT: sxtw x13, w3
+; CHECK-SD-NEXT: sbfx x8, x1, #31, #1
+; CHECK-SD-NEXT: umulh x12, x10, x2
+; CHECK-SD-NEXT: asr x16, x13, #63
+; CHECK-SD-NEXT: mul x14, x9, x2
+; CHECK-SD-NEXT: mul x8, x8, x2
+; CHECK-SD-NEXT: umulh x11, x9, x2
+; CHECK-SD-NEXT: adds x12, x14, x12
+; CHECK-SD-NEXT: mul x17, x10, x13
+; CHECK-SD-NEXT: umulh x15, x10, x13
+; CHECK-SD-NEXT: adc x8, x11, x8
+; CHECK-SD-NEXT: mul x10, x10, x16
+; CHECK-SD-NEXT: asr x11, x8, #63
+; CHECK-SD-NEXT: cmn x17, x12
+; CHECK-SD-NEXT: mul x16, x9, x13
+; CHECK-SD-NEXT: smulh x9, x9, x13
+; CHECK-SD-NEXT: adc x10, x15, x10
+; CHECK-SD-NEXT: asr x12, x10, #63
+; CHECK-SD-NEXT: adds x8, x8, x10
+; CHECK-SD-NEXT: adc x10, x11, x12
+; CHECK-SD-NEXT: adds x0, x16, x8
+; CHECK-SD-NEXT: adc x1, x9, x10
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_i96:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sxtw x8, w1
+; CHECK-GI-NEXT: sxtw x10, w3
+; CHECK-GI-NEXT: umulh x9, x0, x2
+; CHECK-GI-NEXT: umulh x11, x8, x2
+; CHECK-GI-NEXT: asr x14, x10, #63
+; CHECK-GI-NEXT: umulh x12, x0, x10
+; CHECK-GI-NEXT: mul x13, x0, x10
+; CHECK-GI-NEXT: mul x15, x8, x2
+; CHECK-GI-NEXT: add x11, x11, x12
+; CHECK-GI-NEXT: asr x12, x8, #63
+; CHECK-GI-NEXT: madd x11, x0, x14, x11
+; CHECK-GI-NEXT: adds x9, x9, x13
+; CHECK-GI-NEXT: smaddl x8, w8, w10, x11
+; CHECK-GI-NEXT: cset w10, hs
+; CHECK-GI-NEXT: adds x9, x9, x15
+; CHECK-GI-NEXT: madd x8, x12, x2, x8
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: and x10, x10, #0x1
+; CHECK-GI-NEXT: and x11, x11, #0x1
+; CHECK-GI-NEXT: add x10, x10, x11
+; CHECK-GI-NEXT: add x8, x8, x10
+; CHECK-GI-NEXT: asr x10, x8, #63
+; CHECK-GI-NEXT: extr x0, x8, x9, #32
+; CHECK-GI-NEXT: extr x1, x10, x8, #32
+; CHECK-GI-NEXT: ret
+ %r = call i96 @llvm.smulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @smulh_i128(i128 %x, i128 %y) nounwind {
+; CHECK-SD-LABEL: smulh_i128:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: asr x9, x1, #63
+; CHECK-SD-NEXT: umulh x10, x0, x2
+; CHECK-SD-NEXT: asr x13, x3, #63
+; CHECK-SD-NEXT: mul x11, x1, x2
+; CHECK-SD-NEXT: umulh x8, x1, x2
+; CHECK-SD-NEXT: mul x9, x9, x2
+; CHECK-SD-NEXT: adds x10, x11, x10
+; CHECK-SD-NEXT: mul x14, x0, x3
+; CHECK-SD-NEXT: umulh x12, x0, x3
+; CHECK-SD-NEXT: adc x8, x8, x9
+; CHECK-SD-NEXT: mul x13, x0, x13
+; CHECK-SD-NEXT: asr x11, x8, #63
+; CHECK-SD-NEXT: cmn x14, x10
+; CHECK-SD-NEXT: mul x15, x1, x3
+; CHECK-SD-NEXT: smulh x9, x1, x3
+; CHECK-SD-NEXT: adc x10, x12, x13
+; CHECK-SD-NEXT: asr x12, x10, #63
+; CHECK-SD-NEXT: adds x8, x8, x10
+; CHECK-SD-NEXT: adc x10, x11, x12
+; CHECK-SD-NEXT: adds x0, x15, x8
+; CHECK-SD-NEXT: adc x1, x9, x10
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_i128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: asr x8, x1, #63
+; CHECK-GI-NEXT: asr x14, x3, #63
+; CHECK-GI-NEXT: umulh x15, x1, x3
+; CHECK-GI-NEXT: umulh x13, x8, x2
+; CHECK-GI-NEXT: umulh x16, x0, x14
+; CHECK-GI-NEXT: umulh x9, x0, x2
+; CHECK-GI-NEXT: add x13, x13, x15
+; CHECK-GI-NEXT: mul x10, x0, x3
+; CHECK-GI-NEXT: add x13, x13, x16
+; CHECK-GI-NEXT: mul x11, x1, x2
+; CHECK-GI-NEXT: umulh x12, x1, x2
+; CHECK-GI-NEXT: adds x9, x9, x10
+; CHECK-GI-NEXT: umulh x17, x0, x3
+; CHECK-GI-NEXT: madd x13, x0, x14, x13
+; CHECK-GI-NEXT: mul x18, x0, x14
+; CHECK-GI-NEXT: mul x15, x1, x3
+; CHECK-GI-NEXT: madd x10, x1, x14, x13
+; CHECK-GI-NEXT: cset w13, hs
+; CHECK-GI-NEXT: cmn x9, x11
+; CHECK-GI-NEXT: cset w9, hs
+; CHECK-GI-NEXT: adds x11, x12, x17
+; CHECK-GI-NEXT: and x13, x13, #0x1
+; CHECK-GI-NEXT: mul x12, x8, x2
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x11, x11, x18
+; CHECK-GI-NEXT: cset w16, hs
+; CHECK-GI-NEXT: and x9, x9, #0x1
+; CHECK-GI-NEXT: and x14, x14, #0x1
+; CHECK-GI-NEXT: madd x10, x8, x3, x10
+; CHECK-GI-NEXT: and x16, x16, #0x1
+; CHECK-GI-NEXT: adds x11, x11, x15
+; CHECK-GI-NEXT: add x9, x13, x9
+; CHECK-GI-NEXT: add x13, x14, x16
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x11, x11, x12
+; CHECK-GI-NEXT: madd x8, x8, x2, x10
+; CHECK-GI-NEXT: and x10, x14, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x0, x11, x9
+; CHECK-GI-NEXT: add x9, x13, x10
+; CHECK-GI-NEXT: and x10, x12, #0x1
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: and x10, x11, #0x1
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: add x1, x8, x9
+; CHECK-GI-NEXT: ret
+ %r = call i128 @llvm.smulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @smulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; CHECK-LABEL: smulh_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: shrn v0.8b, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = call <8 x i8> @llvm.smulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @smulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; CHECK-LABEL: smulh_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NEXT: ret
+ %r = call <4 x i16> @llvm.smulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @smulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: smulh_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: shrn v0.2s, v0.2d, #32
+; CHECK-NEXT: ret
+ %r = call <2 x i32> @llvm.smulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @smulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; CHECK-SD-LABEL: smulh_v1i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: fmov x8, d1
+; CHECK-SD-NEXT: fmov x9, d0
+; CHECK-SD-NEXT: smulh x8, x9, x8
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_v1i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: smulh x8, x8, x9
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: ret
+ %r = call <1 x i64> @llvm.smulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @smulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; CHECK-LABEL: smulh_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %r = call <16 x i8> @llvm.smulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @smulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; CHECK-LABEL: smulh_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT: ret
+ %r = call <8 x i16> @llvm.smulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @smulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: smulh_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: ret
+ %r = call <4 x i32> @llvm.smulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @smulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; CHECK-SD-LABEL: smulh_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: fmov x10, d1
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mov x8, v1.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: smulh x10, x11, x10
+; CHECK-SD-NEXT: smulh x8, x9, x8
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: mov v0.d[1], x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d2, v0.d[1]
+; CHECK-GI-NEXT: mov d3, v1.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: umulh x10, x8, x9
+; CHECK-GI-NEXT: fmov x11, d2
+; CHECK-GI-NEXT: fmov x12, d3
+; CHECK-GI-NEXT: asr x13, x9, #63
+; CHECK-GI-NEXT: umulh x14, x11, x12
+; CHECK-GI-NEXT: madd x10, x8, x13, x10
+; CHECK-GI-NEXT: asr x13, x12, #63
+; CHECK-GI-NEXT: asr x8, x8, #63
+; CHECK-GI-NEXT: madd x13, x11, x13, x14
+; CHECK-GI-NEXT: asr x11, x11, #63
+; CHECK-GI-NEXT: madd x8, x8, x9, x10
+; CHECK-GI-NEXT: madd x9, x11, x12, x13
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i64> @llvm.smulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @smulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; CHECK-SD-LABEL: smulh_v3i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_v3i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v2.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v3.s[0], v1.s[0]
+; CHECK-GI-NEXT: mov w8, v0.s[2]
+; CHECK-GI-NEXT: mov w9, v1.s[2]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[1]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-GI-NEXT: smull x8, w8, w9
+; CHECK-GI-NEXT: smull v2.2d, v2.2s, v3.2s
+; CHECK-GI-NEXT: asr x8, x8, #32
+; CHECK-GI-NEXT: sshr v2.2d, v2.2d, #32
+; CHECK-GI-NEXT: mov d0, v2.d[1]
+; CHECK-GI-NEXT: fmov x10, d2
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov s0, w10
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: mov v0.s[2], w8
+; CHECK-GI-NEXT: ret
+ %r = call <3 x i32> @llvm.smulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @smulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; CHECK-SD-LABEL: smulh_v2i48:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: shl v1.2d, v1.2d, #16
+; CHECK-SD-NEXT: shl v0.2d, v0.2d, #16
+; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #16
+; CHECK-SD-NEXT: fmov x10, d0
+; CHECK-SD-NEXT: mov x8, v0.d[1]
+; CHECK-SD-NEXT: fmov x11, d1
+; CHECK-SD-NEXT: mov x9, v1.d[1]
+; CHECK-SD-NEXT: smulh x10, x10, x11
+; CHECK-SD-NEXT: smulh x8, x8, x9
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: mov v0.d[1], x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_v2i48:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d2, v0.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: mov d0, v1.d[1]
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: lsl x10, x8, #48
+; CHECK-GI-NEXT: sbfx x8, x8, #16, #32
+; CHECK-GI-NEXT: lsl x11, x9, #48
+; CHECK-GI-NEXT: sbfx x9, x9, #16, #32
+; CHECK-GI-NEXT: fmov x12, d2
+; CHECK-GI-NEXT: fmov x13, d0
+; CHECK-GI-NEXT: extr x10, x8, x10, #48
+; CHECK-GI-NEXT: asr x8, x8, #48
+; CHECK-GI-NEXT: extr x11, x9, x11, #48
+; CHECK-GI-NEXT: asr x9, x9, #48
+; CHECK-GI-NEXT: lsl x14, x12, #48
+; CHECK-GI-NEXT: sbfx x12, x12, #16, #32
+; CHECK-GI-NEXT: lsl x16, x13, #48
+; CHECK-GI-NEXT: sbfx x13, x13, #16, #32
+; CHECK-GI-NEXT: umulh x15, x10, x11
+; CHECK-GI-NEXT: extr x14, x12, x14, #48
+; CHECK-GI-NEXT: asr x12, x12, #48
+; CHECK-GI-NEXT: extr x16, x13, x16, #48
+; CHECK-GI-NEXT: asr x13, x13, #48
+; CHECK-GI-NEXT: madd x9, x10, x9, x15
+; CHECK-GI-NEXT: umulh x15, x14, x16
+; CHECK-GI-NEXT: madd x8, x8, x11, x9
+; CHECK-GI-NEXT: madd x13, x14, x13, x15
+; CHECK-GI-NEXT: mul x10, x10, x11
+; CHECK-GI-NEXT: sxtw x8, w8
+; CHECK-GI-NEXT: madd x9, x12, x16, x13
+; CHECK-GI-NEXT: mul x11, x14, x16
+; CHECK-GI-NEXT: extr x8, x8, x10, #48
+; CHECK-GI-NEXT: sxtw x9, w9
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: extr x9, x9, x11, #48
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i48> @llvm.smulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @smulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; CHECK-SD-LABEL: smulh_v2i128:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: asr x10, x1, #63
+; CHECK-SD-NEXT: asr x8, x5, #63
+; CHECK-SD-NEXT: umulh x11, x0, x4
+; CHECK-SD-NEXT: asr x18, x3, #63
+; CHECK-SD-NEXT: mul x12, x1, x4
+; CHECK-SD-NEXT: umulh x9, x1, x4
+; CHECK-SD-NEXT: mul x10, x10, x4
+; CHECK-SD-NEXT: adds x11, x12, x11
+; CHECK-SD-NEXT: mul x14, x0, x5
+; CHECK-SD-NEXT: umulh x13, x0, x5
+; CHECK-SD-NEXT: adc x9, x9, x10
+; CHECK-SD-NEXT: mul x8, x0, x8
+; CHECK-SD-NEXT: asr x0, x7, #63
+; CHECK-SD-NEXT: cmn x14, x11
+; CHECK-SD-NEXT: asr x11, x9, #63
+; CHECK-SD-NEXT: mul x16, x1, x5
+; CHECK-SD-NEXT: smulh x15, x1, x5
+; CHECK-SD-NEXT: adc x8, x13, x8
+; CHECK-SD-NEXT: umulh x4, x2, x6
+; CHECK-SD-NEXT: asr x12, x8, #63
+; CHECK-SD-NEXT: adds x8, x9, x8
+; CHECK-SD-NEXT: mul x5, x3, x6
+; CHECK-SD-NEXT: adc x9, x11, x12
+; CHECK-SD-NEXT: umulh x17, x3, x6
+; CHECK-SD-NEXT: mul x18, x18, x6
+; CHECK-SD-NEXT: mul x10, x2, x7
+; CHECK-SD-NEXT: mul x6, x2, x0
+; CHECK-SD-NEXT: adds x0, x16, x8
+; CHECK-SD-NEXT: adc x1, x15, x9
+; CHECK-SD-NEXT: adds x9, x5, x4
+; CHECK-SD-NEXT: umulh x13, x2, x7
+; CHECK-SD-NEXT: adc x11, x17, x18
+; CHECK-SD-NEXT: cmn x10, x9
+; CHECK-SD-NEXT: asr x12, x11, #63
+; CHECK-SD-NEXT: mul x8, x3, x7
+; CHECK-SD-NEXT: smulh x9, x3, x7
+; CHECK-SD-NEXT: adc x10, x13, x6
+; CHECK-SD-NEXT: asr x13, x10, #63
+; CHECK-SD-NEXT: adds x10, x11, x10
+; CHECK-SD-NEXT: adc x11, x12, x13
+; CHECK-SD-NEXT: adds x2, x8, x10
+; CHECK-SD-NEXT: adc x3, x9, x11
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: smulh_v2i128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: stp x22, x21, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: asr x8, x1, #63
+; CHECK-GI-NEXT: asr x14, x5, #63
+; CHECK-GI-NEXT: umulh x15, x1, x5
+; CHECK-GI-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: umulh x12, x8, x4
+; CHECK-GI-NEXT: umulh x16, x0, x14
+; CHECK-GI-NEXT: umulh x9, x0, x4
+; CHECK-GI-NEXT: add x12, x12, x15
+; CHECK-GI-NEXT: mul x11, x0, x5
+; CHECK-GI-NEXT: add x12, x12, x16
+; CHECK-GI-NEXT: madd x12, x0, x14, x12
+; CHECK-GI-NEXT: mul x19, x0, x14
+; CHECK-GI-NEXT: adds x9, x9, x11
+; CHECK-GI-NEXT: madd x12, x1, x14, x12
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: and x11, x11, #0x1
+; CHECK-GI-NEXT: mul x10, x1, x4
+; CHECK-GI-NEXT: madd x14, x8, x5, x12
+; CHECK-GI-NEXT: asr x12, x3, #63
+; CHECK-GI-NEXT: umulh x13, x1, x4
+; CHECK-GI-NEXT: cmn x9, x10
+; CHECK-GI-NEXT: mul x18, x1, x5
+; CHECK-GI-NEXT: asr x1, x7, #63
+; CHECK-GI-NEXT: umulh x20, x0, x5
+; CHECK-GI-NEXT: umulh x0, x12, x6
+; CHECK-GI-NEXT: umulh x5, x3, x7
+; CHECK-GI-NEXT: umulh x21, x2, x1
+; CHECK-GI-NEXT: mul x17, x8, x4
+; CHECK-GI-NEXT: add x10, x0, x5
+; CHECK-GI-NEXT: cset w0, hs
+; CHECK-GI-NEXT: adds x13, x13, x20
+; CHECK-GI-NEXT: cset w5, hs
+; CHECK-GI-NEXT: adds x13, x13, x19
+; CHECK-GI-NEXT: umulh x15, x2, x6
+; CHECK-GI-NEXT: add x10, x10, x21
+; CHECK-GI-NEXT: cset w19, hs
+; CHECK-GI-NEXT: adds x13, x13, x18
+; CHECK-GI-NEXT: mul x16, x2, x7
+; CHECK-GI-NEXT: and x5, x5, #0x1
+; CHECK-GI-NEXT: and x19, x19, #0x1
+; CHECK-GI-NEXT: cset w18, hs
+; CHECK-GI-NEXT: add x5, x5, x19
+; CHECK-GI-NEXT: adds x13, x13, x17
+; CHECK-GI-NEXT: madd x10, x2, x1, x10
+; CHECK-GI-NEXT: and x17, x0, #0x1
+; CHECK-GI-NEXT: and x18, x18, #0x1
+; CHECK-GI-NEXT: add x11, x11, x17
+; CHECK-GI-NEXT: add x17, x5, x18
+; CHECK-GI-NEXT: cset w18, hs
+; CHECK-GI-NEXT: mul x22, x3, x6
+; CHECK-GI-NEXT: and x18, x18, #0x1
+; CHECK-GI-NEXT: adds x0, x13, x11
+; CHECK-GI-NEXT: add x13, x17, x18
+; CHECK-GI-NEXT: cset w17, hs
+; CHECK-GI-NEXT: adds x15, x15, x16
+; CHECK-GI-NEXT: umulh x9, x3, x6
+; CHECK-GI-NEXT: cset w16, hs
+; CHECK-GI-NEXT: and x17, x17, #0x1
+; CHECK-GI-NEXT: and x16, x16, #0x1
+; CHECK-GI-NEXT: add x13, x13, x17
+; CHECK-GI-NEXT: mul x20, x2, x1
+; CHECK-GI-NEXT: cmn x15, x22
+; CHECK-GI-NEXT: madd x10, x3, x1, x10
+; CHECK-GI-NEXT: cset w18, hs
+; CHECK-GI-NEXT: and x18, x18, #0x1
+; CHECK-GI-NEXT: umulh x1, x2, x7
+; CHECK-GI-NEXT: mul x11, x3, x7
+; CHECK-GI-NEXT: mul x15, x12, x6
+; CHECK-GI-NEXT: adds x9, x9, x1
+; CHECK-GI-NEXT: madd x10, x12, x7, x10
+; CHECK-GI-NEXT: cset w1, hs
+; CHECK-GI-NEXT: adds x9, x9, x20
+; CHECK-GI-NEXT: cset w2, hs
+; CHECK-GI-NEXT: adds x9, x9, x11
+; CHECK-GI-NEXT: and x1, x1, #0x1
+; CHECK-GI-NEXT: and x2, x2, #0x1
+; CHECK-GI-NEXT: madd x8, x8, x4, x14
+; CHECK-GI-NEXT: add x14, x16, x18
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: adds x9, x9, x15
+; CHECK-GI-NEXT: add x16, x1, x2
+; CHECK-GI-NEXT: madd x10, x12, x6, x10
+; CHECK-GI-NEXT: and x11, x11, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: adds x2, x9, x14
+; CHECK-GI-NEXT: add x9, x16, x11
+; CHECK-GI-NEXT: and x11, x12, #0x1
+; CHECK-GI-NEXT: cset w12, hs
+; CHECK-GI-NEXT: add x9, x9, x11
+; CHECK-GI-NEXT: and x11, x12, #0x1
+; CHECK-GI-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add x9, x9, x11
+; CHECK-GI-NEXT: add x1, x8, x13
+; CHECK-GI-NEXT: add x3, x10, x9
+; CHECK-GI-NEXT: ldp x22, x21, [sp], #32 // 16-byte Folded Reload
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i128> @llvm.smulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-smulh.ll b/llvm/test/CodeGen/AArch64/sve-smulh.ll
new file mode 100644
index 0000000000000..ed8be632022c6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-smulh.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve | FileCheck %s --check-prefix=SVE
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 | FileCheck %s --check-prefix=SVE2
+
+define <vscale x 16 x i8> @smulh_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y) nounwind {
+; SVE-LABEL: smulh_nxv16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.b
+; SVE-NEXT: smulh z0.b, p0/m, z0.b, z1.b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv16i8:
+; SVE2: // %bb.0:
+; SVE2-NEXT: smulh z0.b, z0.b, z1.b
+; SVE2-NEXT: ret
+ %r = call <vscale x 16 x i8> @llvm.smulh(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y)
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @smulh_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y) nounwind {
+; SVE-LABEL: smulh_nxv8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h
+; SVE-NEXT: smulh z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv8i16:
+; SVE2: // %bb.0:
+; SVE2-NEXT: smulh z0.h, z0.h, z1.h
+; SVE2-NEXT: ret
+ %r = call <vscale x 8 x i16> @llvm.smulh(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @smulh_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y) nounwind {
+; SVE-LABEL: smulh_nxv4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: smulh z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv4i32:
+; SVE2: // %bb.0:
+; SVE2-NEXT: smulh z0.s, z0.s, z1.s
+; SVE2-NEXT: ret
+ %r = call <vscale x 4 x i32> @llvm.smulh(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @smulh_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y) nounwind {
+; SVE-LABEL: smulh_nxv2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: smulh z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv2i64:
+; SVE2: // %bb.0:
+; SVE2-NEXT: smulh z0.d, z0.d, z1.d
+; SVE2-NEXT: ret
+ %r = call <vscale x 2 x i64> @llvm.smulh(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
+ ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 3 x i32> @smulh_nxv3i32(<vscale x 3 x i32> %x, <vscale x 3 x i32> %y) nounwind {
+; SVE-LABEL: smulh_nxv3i32:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: smulh z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv3i32:
+; SVE2: // %bb.0:
+; SVE2-NEXT: smulh z0.s, z0.s, z1.s
+; SVE2-NEXT: ret
+ %r = call <vscale x 3 x i32> @llvm.smulh(<vscale x 3 x i32> %x, <vscale x 3 x i32> %y)
+ ret <vscale x 3 x i32> %r
+}
+
+define <vscale x 2 x i48> @smulh_nxv2i48(<vscale x 2 x i48> %x, <vscale x 2 x i48> %y) nounwind {
+; SVE-LABEL: smulh_nxv2i48:
+; SVE: // %bb.0:
+; SVE-NEXT: lsl z1.d, z1.d, #16
+; SVE-NEXT: lsl z0.d, z0.d, #16
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: asr z1.d, z1.d, #16
+; SVE-NEXT: smulh z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: smulh_nxv2i48:
+; SVE2: // %bb.0:
+; SVE2-NEXT: lsl z1.d, z1.d, #16
+; SVE2-NEXT: lsl z0.d, z0.d, #16
+; SVE2-NEXT: asr z1.d, z1.d, #16
+; SVE2-NEXT: smulh z0.d, z0.d, z1.d
+; SVE2-NEXT: ret
+ %r = call <vscale x 2 x i48> @llvm.smulh(<vscale x 2 x i48> %x, <vscale x 2 x i48> %y)
+ ret <vscale x 2 x i48> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-umulh.ll b/llvm/test/CodeGen/AArch64/sve-umulh.ll
new file mode 100644
index 0000000000000..69b4403dd0e49
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-umulh.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve | FileCheck %s --check-prefix=SVE
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 | FileCheck %s --check-prefix=SVE2
+
+define <vscale x 16 x i8> @umulh_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y) nounwind {
+; SVE-LABEL: umulh_nxv16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.b
+; SVE-NEXT: umulh z0.b, p0/m, z0.b, z1.b
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv16i8:
+; SVE2: // %bb.0:
+; SVE2-NEXT: umulh z0.b, z0.b, z1.b
+; SVE2-NEXT: ret
+ %r = call <vscale x 16 x i8> @llvm.umulh(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y)
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 8 x i16> @umulh_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y) nounwind {
+; SVE-LABEL: umulh_nxv8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h
+; SVE-NEXT: umulh z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv8i16:
+; SVE2: // %bb.0:
+; SVE2-NEXT: umulh z0.h, z0.h, z1.h
+; SVE2-NEXT: ret
+ %r = call <vscale x 8 x i16> @llvm.umulh(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 4 x i32> @umulh_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y) nounwind {
+; SVE-LABEL: umulh_nxv4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: umulh z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv4i32:
+; SVE2: // %bb.0:
+; SVE2-NEXT: umulh z0.s, z0.s, z1.s
+; SVE2-NEXT: ret
+ %r = call <vscale x 4 x i32> @llvm.umulh(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @umulh_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y) nounwind {
+; SVE-LABEL: umulh_nxv2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: umulh z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv2i64:
+; SVE2: // %bb.0:
+; SVE2-NEXT: umulh z0.d, z0.d, z1.d
+; SVE2-NEXT: ret
+ %r = call <vscale x 2 x i64> @llvm.umulh(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
+ ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 3 x i32> @umulh_nxv3i32(<vscale x 3 x i32> %x, <vscale x 3 x i32> %y) nounwind {
+; SVE-LABEL: umulh_nxv3i32:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: umulh z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv3i32:
+; SVE2: // %bb.0:
+; SVE2-NEXT: umulh z0.s, z0.s, z1.s
+; SVE2-NEXT: ret
+ %r = call <vscale x 3 x i32> @llvm.umulh(<vscale x 3 x i32> %x, <vscale x 3 x i32> %y)
+ ret <vscale x 3 x i32> %r
+}
+
+define <vscale x 2 x i48> @umulh_nxv2i48(<vscale x 2 x i48> %x, <vscale x 2 x i48> %y) nounwind {
+; SVE-LABEL: umulh_nxv2i48:
+; SVE: // %bb.0:
+; SVE-NEXT: and z1.d, z1.d, #0xffffffffffff
+; SVE-NEXT: lsl z0.d, z0.d, #16
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: umulh z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: ret
+;
+; SVE2-LABEL: umulh_nxv2i48:
+; SVE2: // %bb.0:
+; SVE2-NEXT: and z1.d, z1.d, #0xffffffffffff
+; SVE2-NEXT: lsl z0.d, z0.d, #16
+; SVE2-NEXT: umulh z0.d, z0.d, z1.d
+; SVE2-NEXT: ret
+ %r = call <vscale x 2 x i48> @llvm.umulh(<vscale x 2 x i48> %x, <vscale x 2 x i48> %y)
+ ret <vscale x 2 x i48> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/umulh.ll b/llvm/test/CodeGen/AArch64/umulh.ll
new file mode 100644
index 0000000000000..524762d1b85c7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/umulh.ll
@@ -0,0 +1,434 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i16 @umulh_i16(i16 %x, i16 %y) nounwind {
+; CHECK-SD-LABEL: umulh_i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: and w8, w1, #0xffff
+; CHECK-SD-NEXT: and w9, w0, #0xffff
+; CHECK-SD-NEXT: mul w8, w9, w8
+; CHECK-SD-NEXT: lsr w0, w8, #16
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w8, w0, #0xffff
+; CHECK-GI-NEXT: and w9, w1, #0xffff
+; CHECK-GI-NEXT: mul w8, w8, w9
+; CHECK-GI-NEXT: lsr w0, w8, #16
+; CHECK-GI-NEXT: ret
+ %r = call i16 @llvm.umulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @umulh_i32(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: umulh_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull x8, w0, w1
+; CHECK-NEXT: lsr x0, x8, #32
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.umulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @umulh_i64(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: umulh_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umulh x0, x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.umulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @umulh_i96(i96 %x, i96 %y) nounwind {
+; CHECK-SD-LABEL: umulh_i96:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: extr x9, x1, x0, #32
+; CHECK-SD-NEXT: lsl x10, x0, #32
+; CHECK-SD-NEXT: mov w8, w3
+; CHECK-SD-NEXT: umulh x12, x10, x2
+; CHECK-SD-NEXT: mul x13, x9, x2
+; CHECK-SD-NEXT: umulh x11, x9, x2
+; CHECK-SD-NEXT: umulh x14, x10, x8
+; CHECK-SD-NEXT: mul x10, x10, x8
+; CHECK-SD-NEXT: mul x15, x9, x8
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: adds x9, x13, x12
+; CHECK-SD-NEXT: cinc x11, x11, hs
+; CHECK-SD-NEXT: cmn x10, x9
+; CHECK-SD-NEXT: adcs x9, x11, x14
+; CHECK-SD-NEXT: cset w10, hs
+; CHECK-SD-NEXT: adds x0, x15, x9
+; CHECK-SD-NEXT: adc x1, x8, x10
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_i96:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w9, w3
+; CHECK-GI-NEXT: mov w11, w1
+; CHECK-GI-NEXT: umulh x8, x0, x2
+; CHECK-GI-NEXT: mul x10, x0, x9
+; CHECK-GI-NEXT: umulh x12, x11, x2
+; CHECK-GI-NEXT: umulh x13, x0, x9
+; CHECK-GI-NEXT: adds x8, x8, x10
+; CHECK-GI-NEXT: mul x14, x11, x2
+; CHECK-GI-NEXT: add x10, x12, x13
+; CHECK-GI-NEXT: umaddl x9, w11, w9, x10
+; CHECK-GI-NEXT: cset w10, hs
+; CHECK-GI-NEXT: adds x8, x8, x14
+; CHECK-GI-NEXT: and x10, x10, #0x1
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: and x11, x11, #0x1
+; CHECK-GI-NEXT: add x9, x9, x11
+; CHECK-GI-NEXT: lsr x1, x9, #32
+; CHECK-GI-NEXT: extr x0, x9, x8, #32
+; CHECK-GI-NEXT: ret
+ %r = call i96 @llvm.umulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @umulh_i128(i128 %x, i128 %y) nounwind {
+; CHECK-SD-LABEL: umulh_i128:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umulh x9, x0, x2
+; CHECK-SD-NEXT: mul x10, x1, x2
+; CHECK-SD-NEXT: umulh x8, x1, x2
+; CHECK-SD-NEXT: mul x12, x0, x3
+; CHECK-SD-NEXT: adds x9, x10, x9
+; CHECK-SD-NEXT: umulh x11, x0, x3
+; CHECK-SD-NEXT: cinc x8, x8, hs
+; CHECK-SD-NEXT: mul x13, x1, x3
+; CHECK-SD-NEXT: cmn x12, x9
+; CHECK-SD-NEXT: umulh x10, x1, x3
+; CHECK-SD-NEXT: cinc x9, x11, hs
+; CHECK-SD-NEXT: adds x8, x8, x9
+; CHECK-SD-NEXT: cset w9, hs
+; CHECK-SD-NEXT: adds x0, x13, x8
+; CHECK-SD-NEXT: adc x1, x10, x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_i128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x8, x0, x2
+; CHECK-GI-NEXT: mul x9, x0, x3
+; CHECK-GI-NEXT: mul x10, x1, x2
+; CHECK-GI-NEXT: umulh x11, x1, x2
+; CHECK-GI-NEXT: adds x8, x8, x9
+; CHECK-GI-NEXT: umulh x12, x0, x3
+; CHECK-GI-NEXT: cset w9, hs
+; CHECK-GI-NEXT: cmn x8, x10
+; CHECK-GI-NEXT: and x9, x9, #0x1
+; CHECK-GI-NEXT: mul x13, x1, x3
+; CHECK-GI-NEXT: cset w10, hs
+; CHECK-GI-NEXT: and x10, x10, #0x1
+; CHECK-GI-NEXT: umulh x14, x2, xzr
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: adds x11, x11, x12
+; CHECK-GI-NEXT: umulh x8, x1, x3
+; CHECK-GI-NEXT: cset w10, hs
+; CHECK-GI-NEXT: adds x11, x11, x13
+; CHECK-GI-NEXT: umulh x12, x0, xzr
+; CHECK-GI-NEXT: cset w13, hs
+; CHECK-GI-NEXT: adds x0, x11, x9
+; CHECK-GI-NEXT: and x9, x10, #0x1
+; CHECK-GI-NEXT: and x10, x13, #0x1
+; CHECK-GI-NEXT: cset w11, hs
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: and x10, x11, #0x1
+; CHECK-GI-NEXT: add x8, x14, x8
+; CHECK-GI-NEXT: add x9, x9, x10
+; CHECK-GI-NEXT: add x8, x8, x12
+; CHECK-GI-NEXT: add x1, x8, x9
+; CHECK-GI-NEXT: ret
+ %r = call i128 @llvm.umulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @umulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; CHECK-LABEL: umulh_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: shrn v0.8b, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = call <8 x i8> @llvm.umulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @umulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; CHECK-LABEL: umulh_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-NEXT: ret
+ %r = call <4 x i16> @llvm.umulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @umulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: umulh_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: shrn v0.2s, v0.2d, #32
+; CHECK-NEXT: ret
+ %r = call <2 x i32> @llvm.umulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @umulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; CHECK-SD-LABEL: umulh_v1i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: fmov x8, d1
+; CHECK-SD-NEXT: fmov x9, d0
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_v1i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: ret
+ %r = call <1 x i64> @llvm.umulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @umulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; CHECK-LABEL: umulh_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %r = call <16 x i8> @llvm.umulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @umulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; CHECK-LABEL: umulh_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT: ret
+ %r = call <8 x i16> @llvm.umulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @umulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: umulh_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: ret
+ %r = call <4 x i32> @llvm.umulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @umulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; CHECK-SD-LABEL: umulh_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: fmov x10, d1
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mov x8, v1.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: umulh x10, x11, x10
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: mov v0.d[1], x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: umulh x10, x10, x11
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i64> @llvm.umulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @umulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; CHECK-SD-LABEL: umulh_v3i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_v3i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v2.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v3.s[0], v1.s[0]
+; CHECK-GI-NEXT: mov w8, v0.s[2]
+; CHECK-GI-NEXT: mov w9, v1.s[2]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[1]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-GI-NEXT: umull x8, w8, w9
+; CHECK-GI-NEXT: umull v2.2d, v2.2s, v3.2s
+; CHECK-GI-NEXT: lsr x8, x8, #32
+; CHECK-GI-NEXT: ushr v2.2d, v2.2d, #32
+; CHECK-GI-NEXT: mov d0, v2.d[1]
+; CHECK-GI-NEXT: fmov x10, d2
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov s0, w10
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: mov v0.s[2], w8
+; CHECK-GI-NEXT: ret
+ %r = call <3 x i32> @llvm.umulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @umulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; CHECK-SD-LABEL: umulh_v2i48:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: movi v2.2d, #0x00ffffffffffff
+; CHECK-SD-NEXT: shl v0.2d, v0.2d, #16
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: fmov x10, d1
+; CHECK-SD-NEXT: mov x8, v1.d[1]
+; CHECK-SD-NEXT: umulh x10, x11, x10
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: mov v0.d[1], x8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_v2i48:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: and x9, x9, #0xffffffffffff
+; CHECK-GI-NEXT: and x10, x10, #0xffffffffffff
+; CHECK-GI-NEXT: mul x12, x9, x10
+; CHECK-GI-NEXT: and x8, x8, #0xffffffffffff
+; CHECK-GI-NEXT: umulh x9, x9, x10
+; CHECK-GI-NEXT: and x10, x11, #0xffffffffffff
+; CHECK-GI-NEXT: mul x11, x8, x10
+; CHECK-GI-NEXT: umulh x8, x8, x10
+; CHECK-GI-NEXT: extr x9, x9, x12, #48
+; CHECK-GI-NEXT: fmov d0, x9
+; CHECK-GI-NEXT: extr x8, x8, x11, #48
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i48> @llvm.umulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @umulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; CHECK-SD-LABEL: umulh_v2i128:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umulh x10, x0, x4
+; CHECK-SD-NEXT: mul x11, x1, x4
+; CHECK-SD-NEXT: umulh x9, x1, x4
+; CHECK-SD-NEXT: mul x13, x0, x5
+; CHECK-SD-NEXT: adds x10, x11, x10
+; CHECK-SD-NEXT: umulh x12, x0, x5
+; CHECK-SD-NEXT: cinc x9, x9, hs
+; CHECK-SD-NEXT: mul x15, x1, x5
+; CHECK-SD-NEXT: cmn x13, x10
+; CHECK-SD-NEXT: umulh x14, x1, x5
+; CHECK-SD-NEXT: cinc x11, x12, hs
+; CHECK-SD-NEXT: umulh x16, x2, x6
+; CHECK-SD-NEXT: adds x9, x9, x11
+; CHECK-SD-NEXT: cset w12, hs
+; CHECK-SD-NEXT: adds x0, x15, x9
+; CHECK-SD-NEXT: mul x17, x3, x6
+; CHECK-SD-NEXT: adc x1, x14, x12
+; CHECK-SD-NEXT: umulh x8, x3, x6
+; CHECK-SD-NEXT: mul x10, x2, x7
+; CHECK-SD-NEXT: adds x9, x17, x16
+; CHECK-SD-NEXT: umulh x18, x2, x7
+; CHECK-SD-NEXT: cinc x8, x8, hs
+; CHECK-SD-NEXT: mul x11, x3, x7
+; CHECK-SD-NEXT: cmn x10, x9
+; CHECK-SD-NEXT: umulh x12, x3, x7
+; CHECK-SD-NEXT: cinc x9, x18, hs
+; CHECK-SD-NEXT: adds x8, x8, x9
+; CHECK-SD-NEXT: cset w9, hs
+; CHECK-SD-NEXT: adds x2, x11, x8
+; CHECK-SD-NEXT: adc x3, x12, x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: umulh_v2i128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x8, x0, x4
+; CHECK-GI-NEXT: mul x9, x0, x5
+; CHECK-GI-NEXT: mul x10, x1, x4
+; CHECK-GI-NEXT: umulh x11, x1, x4
+; CHECK-GI-NEXT: adds x8, x8, x9
+; CHECK-GI-NEXT: umulh x12, x0, x5
+; CHECK-GI-NEXT: cset w9, hs
+; CHECK-GI-NEXT: cmn x8, x10
+; CHECK-GI-NEXT: and x8, x9, #0x1
+; CHECK-GI-NEXT: mul x13, x1, x5
+; CHECK-GI-NEXT: cset w9, hs
+; CHECK-GI-NEXT: and x9, x9, #0x1
+; CHECK-GI-NEXT: umulh x16, x2, x6
+; CHECK-GI-NEXT: add x8, x8, x9
+; CHECK-GI-NEXT: adds x11, x11, x12
+; CHECK-GI-NEXT: mul x17, x2, x7
+; CHECK-GI-NEXT: cset w9, hs
+; CHECK-GI-NEXT: adds x11, x11, x13
+; CHECK-GI-NEXT: and x9, x9, #0x1
+; CHECK-GI-NEXT: mul x18, x3, x6
+; CHECK-GI-NEXT: cset w13, hs
+; CHECK-GI-NEXT: adds x8, x11, x8
+; CHECK-GI-NEXT: and x13, x13, #0x1
+; CHECK-GI-NEXT: umulh x14, x4, xzr
+; CHECK-GI-NEXT: add x9, x9, x13
+; CHECK-GI-NEXT: umulh x15, x1, x5
+; CHECK-GI-NEXT: umulh x1, x3, x6
+; CHECK-GI-NEXT: umulh x10, x2, x7
+; CHECK-GI-NEXT: add x14, x14, x15
+; CHECK-GI-NEXT: mul x12, x3, x7
+; CHECK-GI-NEXT: umulh x11, x3, x7
+; CHECK-GI-NEXT: cset w3, hs
+; CHECK-GI-NEXT: adds x16, x16, x17
+; CHECK-GI-NEXT: and x13, x3, #0x1
+; CHECK-GI-NEXT: umulh x17, x0, xzr
+; CHECK-GI-NEXT: cset w0, hs
+; CHECK-GI-NEXT: cmn x16, x18
+; CHECK-GI-NEXT: add x9, x9, x13
+; CHECK-GI-NEXT: cset w16, hs
+; CHECK-GI-NEXT: and x15, x0, #0x1
+; CHECK-GI-NEXT: umulh x13, x6, xzr
+; CHECK-GI-NEXT: and x16, x16, #0x1
+; CHECK-GI-NEXT: adds x10, x1, x10
+; CHECK-GI-NEXT: add x15, x15, x16
+; CHECK-GI-NEXT: cset w16, hs
+; CHECK-GI-NEXT: adds x10, x10, x12
+; CHECK-GI-NEXT: umulh x18, x2, xzr
+; CHECK-GI-NEXT: mov x0, x8
+; CHECK-GI-NEXT: add x12, x14, x17
+; CHECK-GI-NEXT: cset w14, hs
+; CHECK-GI-NEXT: adds x2, x10, x15
+; CHECK-GI-NEXT: and x10, x16, #0x1
+; CHECK-GI-NEXT: and x14, x14, #0x1
+; CHECK-GI-NEXT: cset w15, hs
+; CHECK-GI-NEXT: add x10, x10, x14
+; CHECK-GI-NEXT: and x14, x15, #0x1
+; CHECK-GI-NEXT: add x11, x13, x11
+; CHECK-GI-NEXT: add x10, x10, x14
+; CHECK-GI-NEXT: add x1, x12, x9
+; CHECK-GI-NEXT: add x11, x11, x18
+; CHECK-GI-NEXT: add x3, x11, x10
+; CHECK-GI-NEXT: ret
+ %r = call <2 x i128> @llvm.umulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
diff --git a/llvm/test/CodeGen/RISCV/smulh.ll b/llvm/test/CodeGen/RISCV/smulh.ll
new file mode 100644
index 0000000000000..642162bad17af
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/smulh.ll
@@ -0,0 +1,4864 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV32I
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV64I
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV32M,RV32IM
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV64M,RV64IM
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV32M,CHECK-V,RV32V
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV64M,CHECK-V,RV64V
+
+define i16 @smulh_i16(i16 %x, i16 %y) nounwind {
+; RV32I-LABEL: smulh_i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: slli a0, a0, 16
+; RV32I-NEXT: slli a1, a1, 16
+; RV32I-NEXT: srai a0, a0, 16
+; RV32I-NEXT: srai a1, a1, 16
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srai a0, a0, 16
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: slli a0, a0, 48
+; RV64I-NEXT: slli a1, a1, 48
+; RV64I-NEXT: srai a0, a0, 48
+; RV64I-NEXT: srai a1, a1, 48
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srai a0, a0, 16
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_i16:
+; RV32M: # %bb.0:
+; RV32M-NEXT: slli a1, a1, 16
+; RV32M-NEXT: slli a0, a0, 16
+; RV32M-NEXT: srai a1, a1, 16
+; RV32M-NEXT: srai a0, a0, 16
+; RV32M-NEXT: mul a0, a0, a1
+; RV32M-NEXT: srli a0, a0, 16
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_i16:
+; RV64M: # %bb.0:
+; RV64M-NEXT: slli a1, a1, 48
+; RV64M-NEXT: slli a0, a0, 48
+; RV64M-NEXT: srai a1, a1, 48
+; RV64M-NEXT: srai a0, a0, 48
+; RV64M-NEXT: mul a0, a0, a1
+; RV64M-NEXT: srai a0, a0, 16
+; RV64M-NEXT: ret
+ %r = call i16 @llvm.smulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @smulh_i32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: smulh_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a2, a1
+; RV32I-NEXT: srai a1, a0, 31
+; RV32I-NEXT: srai a3, a2, 31
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sext.w a0, a0
+; RV64I-NEXT: sext.w a1, a1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srai a0, a0, 32
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_i32:
+; RV32M: # %bb.0:
+; RV32M-NEXT: mulh a0, a0, a1
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_i32:
+; RV64M: # %bb.0:
+; RV64M-NEXT: sext.w a1, a1
+; RV64M-NEXT: sext.w a0, a0
+; RV64M-NEXT: mul a0, a0, a1
+; RV64M-NEXT: srli a0, a0, 32
+; RV64M-NEXT: ret
+ %r = call i32 @llvm.smulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @smulh_i64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: smulh_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: srai s4, a1, 31
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, s5, a1
+; RV32I-NEXT: sltu a0, s7, s5
+; RV32I-NEXT: add s6, s6, a0
+; RV32I-NEXT: srai s1, s0, 31
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, a0, s7
+; RV32I-NEXT: sltu a0, s7, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s6, a0
+; RV32I-NEXT: srai a1, s6, 31
+; RV32I-NEXT: srai a0, a0, 31
+; RV32I-NEXT: sltu a2, s3, s6
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s5, a0, a2
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: add a1, a1, s5
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a1, a1, a0
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a2, a1
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: srai a3, a2, 63
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_i64:
+; RV32M: # %bb.0:
+; RV32M-NEXT: mulhu a4, a0, a2
+; RV32M-NEXT: mul a5, a1, a2
+; RV32M-NEXT: mul a6, a3, a0
+; RV32M-NEXT: mulhsu a2, a1, a2
+; RV32M-NEXT: mulhsu a0, a3, a0
+; RV32M-NEXT: mulh a7, a1, a3
+; RV32M-NEXT: add a4, a5, a4
+; RV32M-NEXT: mul a1, a1, a3
+; RV32M-NEXT: add a3, a6, a4
+; RV32M-NEXT: sltu a4, a4, a5
+; RV32M-NEXT: sltu a3, a3, a6
+; RV32M-NEXT: add a2, a2, a4
+; RV32M-NEXT: add a0, a0, a3
+; RV32M-NEXT: add a3, a2, a0
+; RV32M-NEXT: srai a4, a2, 31
+; RV32M-NEXT: srai a0, a0, 31
+; RV32M-NEXT: sltu a2, a3, a2
+; RV32M-NEXT: add a0, a4, a0
+; RV32M-NEXT: add a2, a0, a2
+; RV32M-NEXT: add a0, a1, a3
+; RV32M-NEXT: add a2, a7, a2
+; RV32M-NEXT: sltu a1, a0, a1
+; RV32M-NEXT: add a1, a2, a1
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_i64:
+; RV64M: # %bb.0:
+; RV64M-NEXT: mulh a0, a0, a1
+; RV64M-NEXT: ret
+ %r = call i64 @llvm.smulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @smulh_i96(i96 %x, i96 %y) nounwind {
+; RV32I-LABEL: smulh_i96:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw a0, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s6, 0(a1)
+; RV32I-NEXT: lw s1, 4(a1)
+; RV32I-NEXT: lw s4, 8(a1)
+; RV32I-NEXT: lw a0, 8(a2)
+; RV32I-NEXT: lw s8, 4(a2)
+; RV32I-NEXT: lw s9, 0(a2)
+; RV32I-NEXT: sw a0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: srai s3, a0, 31
+; RV32I-NEXT: srai s7, s4, 31
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s9
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw a0, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s10, a1
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s9
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s10
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add s0, a1, a0
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, s0, a0
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s10, a0
+; RV32I-NEXT: mv s11, a1
+; RV32I-NEXT: add s5, a0, s1
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: sw s7, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a3, s7
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s1, s0
+; RV32I-NEXT: sltu a3, s5, s10
+; RV32I-NEXT: add a2, s11, a2
+; RV32I-NEXT: add s0, s5, a0
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sltu s1, s0, s5
+; RV32I-NEXT: add a1, a2, a1
+; RV32I-NEXT: add s1, a1, s1
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s9
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s10, a0
+; RV32I-NEXT: mv s11, a1
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s9
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, s10, a1
+; RV32I-NEXT: sltu a0, s5, s10
+; RV32I-NEXT: add s11, s11, a0
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s5, s11, a0
+; RV32I-NEXT: sltu s9, s5, s11
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: add a1, a1, s9
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s11, a1, s5
+; RV32I-NEXT: add s5, s2, a0
+; RV32I-NEXT: sltu a0, s11, a1
+; RV32I-NEXT: add s5, s5, a0
+; RV32I-NEXT: beq s5, s2, .LBB3_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu a0, s5, s2
+; RV32I-NEXT: .LBB3_2:
+; RV32I-NEXT: add s2, s0, a0
+; RV32I-NEXT: sltu s10, s2, s0
+; RV32I-NEXT: add s1, s1, s10
+; RV32I-NEXT: sw s1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a0
+; RV32I-NEXT: add s0, s7, a1
+; RV32I-NEXT: sltu a0, s0, s7
+; RV32I-NEXT: add s10, s8, a0
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s4, s10, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: add s1, a0, s4
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: mv a3, s6
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s4, s10
+; RV32I-NEXT: add a0, s1, a0
+; RV32I-NEXT: add a2, s8, a2
+; RV32I-NEXT: sltu a3, a0, s1
+; RV32I-NEXT: sltu a4, s1, s7
+; RV32I-NEXT: add a4, a2, a4
+; RV32I-NEXT: add s11, s9, s11
+; RV32I-NEXT: add s5, s0, s5
+; RV32I-NEXT: sltu a2, s11, s9
+; RV32I-NEXT: add a4, a4, a1
+; RV32I-NEXT: add a1, s5, a2
+; RV32I-NEXT: add s5, a4, a3
+; RV32I-NEXT: beq a1, s0, .LBB3_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu a2, a1, s0
+; RV32I-NEXT: .LBB3_4:
+; RV32I-NEXT: add a2, a0, a2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s5, s5, a0
+; RV32I-NEXT: add s0, s2, a2
+; RV32I-NEXT: lw s11, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s11, s11, s5
+; RV32I-NEXT: sltu s1, s0, s2
+; RV32I-NEXT: add s11, s11, s1
+; RV32I-NEXT: lw s2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s8, a0
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: lw s10, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, s8, s0
+; RV32I-NEXT: add a2, a0, s9
+; RV32I-NEXT: sltu s8, s0, s8
+; RV32I-NEXT: add s2, a2, s8
+; RV32I-NEXT: add s2, s2, s11
+; RV32I-NEXT: beq s2, a2, .LBB3_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu s8, s2, a2
+; RV32I-NEXT: .LBB3_6:
+; RV32I-NEXT: lw a3, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: beq s11, a3, .LBB3_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s1, s11, a3
+; RV32I-NEXT: .LBB3_8:
+; RV32I-NEXT: srai a3, a3, 31
+; RV32I-NEXT: srai a4, s5, 31
+; RV32I-NEXT: sltu a5, s9, s7
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a5, s6, a5
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, a3, s1
+; RV32I-NEXT: add s5, a5, a0
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: lw a2, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: lw a2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, s10
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s1, s1, s8
+; RV32I-NEXT: add a0, a0, s4
+; RV32I-NEXT: add a0, s6, a0
+; RV32I-NEXT: add a0, a0, s5
+; RV32I-NEXT: add a0, a0, s1
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw s0, 0(a1)
+; RV32I-NEXT: sw s2, 4(a1)
+; RV32I-NEXT: sw a0, 8(a1)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_i96:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a2
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: mv s2, a0
+; RV64I-NEXT: sext.w s3, a3
+; RV64I-NEXT: slli s4, a0, 32
+; RV64I-NEXT: srai s5, s3, 63
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a0, s4
+; RV64I-NEXT: mv a2, s3
+; RV64I-NEXT: mv a3, s5
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv s7, a1
+; RV64I-NEXT: mv a0, s4
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s4, a1
+; RV64I-NEXT: srli a0, s2, 32
+; RV64I-NEXT: slli a1, s1, 32
+; RV64I-NEXT: or s2, a1, a0
+; RV64I-NEXT: sraiw s1, s1, 31
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s4, a0, s4
+; RV64I-NEXT: add a2, s6, s4
+; RV64I-NEXT: sltu a0, s4, a0
+; RV64I-NEXT: sltu a2, a2, s6
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add a2, s7, a2
+; RV64I-NEXT: add s0, a0, a2
+; RV64I-NEXT: srai a2, a2, 63
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: sltu a0, s0, a0
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: add s4, a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: mv a2, s3
+; RV64I-NEXT: mv a3, s5
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add a2, a0, s0
+; RV64I-NEXT: add a1, a1, s4
+; RV64I-NEXT: sltu a0, a2, a0
+; RV64I-NEXT: add a1, a1, a0
+; RV64I-NEXT: mv a0, a2
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_i96:
+; RV32M: # %bb.0:
+; RV32M-NEXT: lw a7, 0(a2)
+; RV32M-NEXT: lw a4, 4(a1)
+; RV32M-NEXT: lw a3, 8(a1)
+; RV32M-NEXT: lw a6, 0(a1)
+; RV32M-NEXT: lw t0, 4(a2)
+; RV32M-NEXT: lw a1, 8(a2)
+; RV32M-NEXT: mulhu t6, a4, a7
+; RV32M-NEXT: mul t1, a3, a7
+; RV32M-NEXT: mulhu a2, a6, a7
+; RV32M-NEXT: mul a5, a6, t0
+; RV32M-NEXT: mul t3, a4, t0
+; RV32M-NEXT: mulhu t2, a6, t0
+; RV32M-NEXT: mul t5, a4, a7
+; RV32M-NEXT: add t6, t1, t6
+; RV32M-NEXT: add a2, a5, a2
+; RV32M-NEXT: add t4, t3, t6
+; RV32M-NEXT: sltu a5, a2, a5
+; RV32M-NEXT: add a5, t2, a5
+; RV32M-NEXT: add a2, t5, a2
+; RV32M-NEXT: add a5, t4, a5
+; RV32M-NEXT: sltu t2, a2, t5
+; RV32M-NEXT: add t5, a5, t2
+; RV32M-NEXT: srai a2, a1, 31
+; RV32M-NEXT: srai a5, a3, 31
+; RV32M-NEXT: beq t5, t4, .LBB3_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu t2, t5, t4
+; RV32M-NEXT: .LBB3_2:
+; RV32M-NEXT: addi sp, sp, -32
+; RV32M-NEXT: sw s0, 28(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s1, 24(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s2, 20(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s3, 16(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s4, 12(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s5, 8(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s6, 4(sp) # 4-byte Folded Spill
+; RV32M-NEXT: mulhu s0, a3, a7
+; RV32M-NEXT: mulhu s1, a4, t0
+; RV32M-NEXT: mul s2, a6, a1
+; RV32M-NEXT: sltu t6, t6, t1
+; RV32M-NEXT: mul s3, a3, t0
+; RV32M-NEXT: mulhu s4, a6, a1
+; RV32M-NEXT: mul t1, a2, a6
+; RV32M-NEXT: mul s5, a7, a5
+; RV32M-NEXT: seqz s6, t5
+; RV32M-NEXT: sltu t3, t4, t3
+; RV32M-NEXT: addi s6, s6, -1
+; RV32M-NEXT: add t6, s0, t6
+; RV32M-NEXT: add t3, s1, t3
+; RV32M-NEXT: add t5, s2, t5
+; RV32M-NEXT: add t3, t6, t3
+; RV32M-NEXT: sltu t4, t5, s2
+; RV32M-NEXT: add t5, s3, t3
+; RV32M-NEXT: and t4, s6, t4
+; RV32M-NEXT: add s4, t1, s4
+; RV32M-NEXT: add s0, t5, s5
+; RV32M-NEXT: mulhu s1, a3, t0
+; RV32M-NEXT: add t2, s0, t2
+; RV32M-NEXT: mulhu s2, a7, a5
+; RV32M-NEXT: add t4, s4, t4
+; RV32M-NEXT: mul t0, t0, a5
+; RV32M-NEXT: add s6, t2, t4
+; RV32M-NEXT: sltu a7, s6, t2
+; RV32M-NEXT: sltu s3, t5, s3
+; RV32M-NEXT: sltu t5, s0, t5
+; RV32M-NEXT: sltu t3, t3, t6
+; RV32M-NEXT: mulhu t6, a6, a2
+; RV32M-NEXT: add t3, s1, t3
+; RV32M-NEXT: add s2, s2, s5
+; RV32M-NEXT: add t3, t3, s3
+; RV32M-NEXT: add t0, s2, t0
+; RV32M-NEXT: add t0, t3, t0
+; RV32M-NEXT: mulhu t3, a4, a1
+; RV32M-NEXT: mul a6, a3, a1
+; RV32M-NEXT: mul s1, a4, a1
+; RV32M-NEXT: add t5, t0, t5
+; RV32M-NEXT: mul t0, a2, a4
+; RV32M-NEXT: sltu s2, s4, t1
+; RV32M-NEXT: add t1, t6, t1
+; RV32M-NEXT: sltu t4, t4, s4
+; RV32M-NEXT: add t1, t1, s2
+; RV32M-NEXT: sltu t2, t2, s0
+; RV32M-NEXT: add t4, t1, t4
+; RV32M-NEXT: add t5, t5, t2
+; RV32M-NEXT: add t2, t4, a7
+; RV32M-NEXT: add t6, a6, t3
+; RV32M-NEXT: add t1, s1, s6
+; RV32M-NEXT: add s0, t0, t6
+; RV32M-NEXT: sltu t3, t1, s1
+; RV32M-NEXT: add s1, t5, t2
+; RV32M-NEXT: add t2, s0, t3
+; RV32M-NEXT: add t2, t2, s1
+; RV32M-NEXT: beq t2, s0, .LBB3_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu t3, t2, s0
+; RV32M-NEXT: .LBB3_4:
+; RV32M-NEXT: beq s1, t5, .LBB3_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu a7, s1, t5
+; RV32M-NEXT: .LBB3_6:
+; RV32M-NEXT: mulhu s1, a3, a1
+; RV32M-NEXT: mul a1, a1, a5
+; RV32M-NEXT: mulhu a4, a4, a2
+; RV32M-NEXT: mul a2, a3, a2
+; RV32M-NEXT: srai a3, t5, 31
+; RV32M-NEXT: srai a5, t4, 31
+; RV32M-NEXT: sltu a6, t6, a6
+; RV32M-NEXT: sltu t4, s0, t0
+; RV32M-NEXT: add a3, a3, a5
+; RV32M-NEXT: add a3, a3, a7
+; RV32M-NEXT: add a6, s1, a6
+; RV32M-NEXT: add a4, a4, t4
+; RV32M-NEXT: add a1, t0, a1
+; RV32M-NEXT: add a4, a6, a4
+; RV32M-NEXT: add a1, a2, a1
+; RV32M-NEXT: add a1, a1, a4
+; RV32M-NEXT: add a3, a3, t3
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: sw t1, 0(a0)
+; RV32M-NEXT: sw t2, 4(a0)
+; RV32M-NEXT: sw a1, 8(a0)
+; RV32M-NEXT: lw s0, 28(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s1, 24(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s2, 20(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s3, 16(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s4, 12(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s5, 8(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s6, 4(sp) # 4-byte Folded Reload
+; RV32M-NEXT: addi sp, sp, 32
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_i96:
+; RV64M: # %bb.0:
+; RV64M-NEXT: sext.w a3, a3
+; RV64M-NEXT: srli a4, a0, 32
+; RV64M-NEXT: slli a1, a1, 32
+; RV64M-NEXT: slli a0, a0, 32
+; RV64M-NEXT: or a1, a1, a4
+; RV64M-NEXT: mulhu a4, a0, a2
+; RV64M-NEXT: mul a5, a1, a2
+; RV64M-NEXT: mul a6, a3, a0
+; RV64M-NEXT: mulhsu a0, a3, a0
+; RV64M-NEXT: mulhsu a2, a1, a2
+; RV64M-NEXT: mulh a7, a1, a3
+; RV64M-NEXT: add a4, a5, a4
+; RV64M-NEXT: mul a1, a1, a3
+; RV64M-NEXT: add a3, a6, a4
+; RV64M-NEXT: sltu a4, a4, a5
+; RV64M-NEXT: sltu a3, a3, a6
+; RV64M-NEXT: add a2, a2, a4
+; RV64M-NEXT: add a0, a0, a3
+; RV64M-NEXT: add a3, a2, a0
+; RV64M-NEXT: srai a4, a2, 63
+; RV64M-NEXT: srai a0, a0, 63
+; RV64M-NEXT: sltu a2, a3, a2
+; RV64M-NEXT: add a0, a4, a0
+; RV64M-NEXT: add a2, a0, a2
+; RV64M-NEXT: add a0, a1, a3
+; RV64M-NEXT: add a2, a7, a2
+; RV64M-NEXT: sltu a1, a0, a1
+; RV64M-NEXT: add a1, a2, a1
+; RV64M-NEXT: ret
+ %r = call i96 @llvm.smulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @smulh_i128(i128 %x, i128 %y) nounwind {
+; RV32I-LABEL: smulh_i128:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -96
+; RV32I-NEXT: sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s10, 0(a2)
+; RV32I-NEXT: lw s8, 4(a2)
+; RV32I-NEXT: lw a3, 8(a2)
+; RV32I-NEXT: sw a3, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a2, 12(a2)
+; RV32I-NEXT: sw a2, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s7, 0(a1)
+; RV32I-NEXT: lw s6, 4(a1)
+; RV32I-NEXT: lw s0, 8(a1)
+; RV32I-NEXT: lw s9, 12(a1)
+; RV32I-NEXT: sw a0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw a0, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s5
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add s1, a1, a0
+; RV32I-NEXT: sw s0, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s1, a0
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s11, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: add s4, a0, s3
+; RV32I-NEXT: sw s9, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: srai a2, s9, 31
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: sw a2, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s3, s1
+; RV32I-NEXT: sltu a3, s4, s11
+; RV32I-NEXT: add a2, s0, a2
+; RV32I-NEXT: add s11, s4, a0
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sltu s1, s11, s4
+; RV32I-NEXT: add a1, a2, a1
+; RV32I-NEXT: add s1, a1, s1
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s3, a1, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s3, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s0
+; RV32I-NEXT: sltu a2, s0, s3
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s5, a1, s5
+; RV32I-NEXT: add s4, s2, a0
+; RV32I-NEXT: sw s5, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, s5, a1
+; RV32I-NEXT: add s4, s4, a0
+; RV32I-NEXT: beq s4, s2, .LBB4_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu a0, s4, s2
+; RV32I-NEXT: .LBB4_2:
+; RV32I-NEXT: add s2, s11, a0
+; RV32I-NEXT: sltu s11, s2, s11
+; RV32I-NEXT: add s1, s1, s11
+; RV32I-NEXT: sw s1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s11, a1, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s1, a0, s0
+; RV32I-NEXT: sltu a0, s1, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s11, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s10, a1
+; RV32I-NEXT: add s0, a0, s3
+; RV32I-NEXT: srai s8, s8, 31
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: mv a3, s6
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s0, s5
+; RV32I-NEXT: add a0, s0, a0
+; RV32I-NEXT: sltu a3, s3, s11
+; RV32I-NEXT: sltu a4, a0, s0
+; RV32I-NEXT: add a3, s10, a3
+; RV32I-NEXT: add a3, a3, a2
+; RV32I-NEXT: lw a2, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a2, s9, a2
+; RV32I-NEXT: add s4, s1, s4
+; RV32I-NEXT: sltu a2, a2, s9
+; RV32I-NEXT: add a3, a3, a1
+; RV32I-NEXT: add a1, s4, a2
+; RV32I-NEXT: add s4, a3, a4
+; RV32I-NEXT: beq a1, s1, .LBB4_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu a2, a1, s1
+; RV32I-NEXT: .LBB4_4:
+; RV32I-NEXT: add a2, a0, a2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s4, s4, a0
+; RV32I-NEXT: add s5, s2, a2
+; RV32I-NEXT: lw s11, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s10, s11, s4
+; RV32I-NEXT: sltu s1, s5, s2
+; RV32I-NEXT: add s10, s10, s1
+; RV32I-NEXT: lw s2, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s3, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: lw a0, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: add s7, a0, s9
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, s0, s5
+; RV32I-NEXT: add a2, a0, s7
+; RV32I-NEXT: sw s5, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu s5, s5, s0
+; RV32I-NEXT: add s2, a2, s5
+; RV32I-NEXT: add s2, s2, s10
+; RV32I-NEXT: beq s2, a2, .LBB4_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu s5, s2, a2
+; RV32I-NEXT: .LBB4_6:
+; RV32I-NEXT: sw s2, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: beq s10, s11, .LBB4_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s1, s10, s11
+; RV32I-NEXT: .LBB4_8:
+; RV32I-NEXT: srai s10, s11, 31
+; RV32I-NEXT: srai a3, s4, 31
+; RV32I-NEXT: sltu a4, s7, s3
+; RV32I-NEXT: add s7, s10, a3
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s6, s6, a4
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, s7, s1
+; RV32I-NEXT: add s11, s6, a0
+; RV32I-NEXT: lw s9, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s3, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: sw a1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s0, a0, s11
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: lw a2, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: lw a2, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, s9
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s7, s10
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: sltu a3, s1, s7
+; RV32I-NEXT: add a4, s0, s3
+; RV32I-NEXT: add a2, s7, a2
+; RV32I-NEXT: add s1, a4, s1
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: add s5, s1, s5
+; RV32I-NEXT: sltu a3, a4, s0
+; RV32I-NEXT: sltu a5, s11, s6
+; RV32I-NEXT: sltu a6, s0, s2
+; RV32I-NEXT: lw a7, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a5, a7, a5
+; RV32I-NEXT: add a1, a1, s4
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a5, a5, a6
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add a0, a5, a0
+; RV32I-NEXT: add a2, a3, a2
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: sltu a1, s1, a4
+; RV32I-NEXT: sltu a2, s5, s1
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: lw a1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a2, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a2, 0(a1)
+; RV32I-NEXT: lw a2, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a2, 4(a1)
+; RV32I-NEXT: sw s5, 8(a1)
+; RV32I-NEXT: sw a0, 12(a1)
+; RV32I-NEXT: lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 96
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_i128:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a3
+; RV64I-NEXT: mv s1, a2
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: mv s3, a0
+; RV64I-NEXT: srai s4, a1, 63
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s5, a0
+; RV64I-NEXT: mv s6, a1
+; RV64I-NEXT: mv a0, s3
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s7, s5, a1
+; RV64I-NEXT: sltu a0, s7, s5
+; RV64I-NEXT: add s6, s6, a0
+; RV64I-NEXT: srai s1, s0, 63
+; RV64I-NEXT: mv a0, s3
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: mv a3, s1
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s7, a0, s7
+; RV64I-NEXT: sltu a0, s7, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s3, s6, a0
+; RV64I-NEXT: srai a1, s6, 63
+; RV64I-NEXT: srai a0, a0, 63
+; RV64I-NEXT: sltu a2, s3, s6
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s5, a0, a2
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: mv a3, s1
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add a2, a0, s3
+; RV64I-NEXT: add a1, a1, s5
+; RV64I-NEXT: sltu a0, a2, a0
+; RV64I-NEXT: add a1, a1, a0
+; RV64I-NEXT: mv a0, a2
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_i128:
+; RV32M: # %bb.0:
+; RV32M-NEXT: addi sp, sp, -32
+; RV32M-NEXT: sw s0, 28(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s1, 24(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s2, 20(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s3, 16(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s4, 12(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s5, 8(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s6, 4(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s7, 0(sp) # 4-byte Folded Spill
+; RV32M-NEXT: lw a3, 8(a1)
+; RV32M-NEXT: lw a4, 12(a1)
+; RV32M-NEXT: lw a6, 0(a2)
+; RV32M-NEXT: lw t1, 4(a2)
+; RV32M-NEXT: lw a5, 8(a2)
+; RV32M-NEXT: lw a2, 12(a2)
+; RV32M-NEXT: mulhu t2, a3, a6
+; RV32M-NEXT: mul t3, a4, a6
+; RV32M-NEXT: mul t4, a3, t1
+; RV32M-NEXT: lw a7, 0(a1)
+; RV32M-NEXT: lw t0, 4(a1)
+; RV32M-NEXT: mulhu t6, a4, a6
+; RV32M-NEXT: mulhu s0, a3, t1
+; RV32M-NEXT: mul s1, a4, t1
+; RV32M-NEXT: srai a1, a4, 31
+; RV32M-NEXT: mulhu s2, a4, t1
+; RV32M-NEXT: mulhu s3, a7, a6
+; RV32M-NEXT: mul s4, a6, a1
+; RV32M-NEXT: mulhu s5, a6, a1
+; RV32M-NEXT: add t2, t3, t2
+; RV32M-NEXT: mul s6, t1, a1
+; RV32M-NEXT: add t5, t4, t2
+; RV32M-NEXT: sltu t2, t2, t3
+; RV32M-NEXT: sltu t3, t5, t4
+; RV32M-NEXT: add t2, t6, t2
+; RV32M-NEXT: add t3, s0, t3
+; RV32M-NEXT: mul t4, t0, a6
+; RV32M-NEXT: add t3, t2, t3
+; RV32M-NEXT: mul t6, a7, t1
+; RV32M-NEXT: add s0, s1, t3
+; RV32M-NEXT: sltu t2, t3, t2
+; RV32M-NEXT: sltu t3, s0, s1
+; RV32M-NEXT: add t2, s2, t2
+; RV32M-NEXT: mulhu s1, t0, a6
+; RV32M-NEXT: add s5, s5, s4
+; RV32M-NEXT: mulhu s2, a7, t1
+; RV32M-NEXT: add s7, t2, t3
+; RV32M-NEXT: add s5, s5, s6
+; RV32M-NEXT: add s3, t4, s3
+; RV32M-NEXT: mul t2, t0, t1
+; RV32M-NEXT: sltu t3, s3, t4
+; RV32M-NEXT: mulhu t1, t0, t1
+; RV32M-NEXT: add s3, t6, s3
+; RV32M-NEXT: mul a6, a3, a6
+; RV32M-NEXT: add s1, s1, t3
+; RV32M-NEXT: sltu t3, s3, t6
+; RV32M-NEXT: add s2, s2, t3
+; RV32M-NEXT: add t3, s0, s4
+; RV32M-NEXT: sltu s0, t3, s0
+; RV32M-NEXT: add s2, s1, s2
+; RV32M-NEXT: sltu t4, s2, s1
+; RV32M-NEXT: add s2, t2, s2
+; RV32M-NEXT: sltu t2, s2, t2
+; RV32M-NEXT: add t1, t1, t4
+; RV32M-NEXT: add t1, t1, t2
+; RV32M-NEXT: add t2, a6, s2
+; RV32M-NEXT: add t1, t5, t1
+; RV32M-NEXT: sltu t6, t2, a6
+; RV32M-NEXT: add s5, s7, s5
+; RV32M-NEXT: add t4, t1, t6
+; RV32M-NEXT: add t1, s5, s0
+; RV32M-NEXT: beq t4, t5, .LBB4_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu t6, t4, t5
+; RV32M-NEXT: .LBB4_2:
+; RV32M-NEXT: mulhu a6, a7, a5
+; RV32M-NEXT: mul t5, t0, a5
+; RV32M-NEXT: mulhu s0, t0, a5
+; RV32M-NEXT: mul s1, a7, a2
+; RV32M-NEXT: mulhu s2, a7, a2
+; RV32M-NEXT: mul s3, t0, a2
+; RV32M-NEXT: add s4, t5, a6
+; RV32M-NEXT: srai a6, a2, 31
+; RV32M-NEXT: sltu t5, s4, t5
+; RV32M-NEXT: mulhu s5, t0, a2
+; RV32M-NEXT: add s0, s0, t5
+; RV32M-NEXT: mul s6, a6, a7
+; RV32M-NEXT: add t5, s1, s4
+; RV32M-NEXT: mul s4, a6, t0
+; RV32M-NEXT: sltu t0, t5, s1
+; RV32M-NEXT: add s2, s2, t0
+; RV32M-NEXT: mulhu s1, a6, a7
+; RV32M-NEXT: add t0, t3, t6
+; RV32M-NEXT: mul t6, a7, a5
+; RV32M-NEXT: sltu t3, t0, t3
+; RV32M-NEXT: add s2, s0, s2
+; RV32M-NEXT: sltu a7, s2, s0
+; RV32M-NEXT: add s2, s3, s2
+; RV32M-NEXT: add s5, s5, a7
+; RV32M-NEXT: add a7, s2, s6
+; RV32M-NEXT: sltu s0, s2, s3
+; RV32M-NEXT: sltu s2, a7, s2
+; RV32M-NEXT: add s0, s5, s0
+; RV32M-NEXT: add s1, s1, s4
+; RV32M-NEXT: add s1, s1, s6
+; RV32M-NEXT: add t2, t6, t2
+; RV32M-NEXT: add t4, t5, t4
+; RV32M-NEXT: sltu t2, t2, t6
+; RV32M-NEXT: add t6, s0, s1
+; RV32M-NEXT: add t4, t4, t2
+; RV32M-NEXT: add t3, t1, t3
+; RV32M-NEXT: add t6, t6, s2
+; RV32M-NEXT: beq t4, t5, .LBB4_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu t2, t4, t5
+; RV32M-NEXT: .LBB4_4:
+; RV32M-NEXT: mulhu s0, a3, a5
+; RV32M-NEXT: mul t4, a4, a5
+; RV32M-NEXT: mul t5, a3, a2
+; RV32M-NEXT: mul t1, a3, a5
+; RV32M-NEXT: add t2, a7, t2
+; RV32M-NEXT: sltu a7, t2, a7
+; RV32M-NEXT: add t6, t6, a7
+; RV32M-NEXT: add a7, t0, t2
+; RV32M-NEXT: add s2, t3, t6
+; RV32M-NEXT: sltu t2, a7, t0
+; RV32M-NEXT: add s2, s2, t2
+; RV32M-NEXT: add s0, t4, s0
+; RV32M-NEXT: add s1, t5, s0
+; RV32M-NEXT: add a7, t1, a7
+; RV32M-NEXT: add t0, s1, s2
+; RV32M-NEXT: sltu t1, a7, t1
+; RV32M-NEXT: add t0, t0, t1
+; RV32M-NEXT: beq t0, s1, .LBB4_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu t1, t0, s1
+; RV32M-NEXT: .LBB4_6:
+; RV32M-NEXT: beq s2, t3, .LBB4_8
+; RV32M-NEXT: # %bb.7:
+; RV32M-NEXT: sltu t2, s2, t3
+; RV32M-NEXT: .LBB4_8:
+; RV32M-NEXT: mulhu s2, a4, a5
+; RV32M-NEXT: mulhu s3, a3, a2
+; RV32M-NEXT: srai t3, t3, 31
+; RV32M-NEXT: mul s4, a4, a2
+; RV32M-NEXT: mul s5, a5, a1
+; RV32M-NEXT: mul s6, a6, a3
+; RV32M-NEXT: srai t6, t6, 31
+; RV32M-NEXT: sltu t4, s0, t4
+; RV32M-NEXT: sltu t5, s1, t5
+; RV32M-NEXT: mulhu s0, a4, a2
+; RV32M-NEXT: mulhu a5, a5, a1
+; RV32M-NEXT: mul a4, a6, a4
+; RV32M-NEXT: add t4, s2, t4
+; RV32M-NEXT: add t5, s3, t5
+; RV32M-NEXT: add t6, t3, t6
+; RV32M-NEXT: add t5, t4, t5
+; RV32M-NEXT: add s1, s6, s5
+; RV32M-NEXT: add s2, s4, t5
+; RV32M-NEXT: add t2, t6, t2
+; RV32M-NEXT: add s3, s2, s1
+; RV32M-NEXT: mulhu a3, a6, a3
+; RV32M-NEXT: add a6, s3, t2
+; RV32M-NEXT: sltu t3, t6, t3
+; RV32M-NEXT: mul a1, a2, a1
+; RV32M-NEXT: add t3, t6, t3
+; RV32M-NEXT: sltu a2, t2, t6
+; RV32M-NEXT: add t1, a6, t1
+; RV32M-NEXT: add a2, t3, a2
+; RV32M-NEXT: sltu t2, s3, s2
+; RV32M-NEXT: sltu t3, t5, t4
+; RV32M-NEXT: sltu t4, s2, s4
+; RV32M-NEXT: add t3, s0, t3
+; RV32M-NEXT: add a5, a5, s5
+; RV32M-NEXT: add a3, a3, a4
+; RV32M-NEXT: add a1, a5, a1
+; RV32M-NEXT: add a3, a3, s6
+; RV32M-NEXT: add a1, a3, a1
+; RV32M-NEXT: sltu a3, s1, s6
+; RV32M-NEXT: add t3, t3, t4
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: add a1, t3, a1
+; RV32M-NEXT: add a2, t2, a2
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: sltu a2, a6, s3
+; RV32M-NEXT: sltu a3, t1, a6
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: sw a7, 0(a0)
+; RV32M-NEXT: sw t0, 4(a0)
+; RV32M-NEXT: sw t1, 8(a0)
+; RV32M-NEXT: sw a1, 12(a0)
+; RV32M-NEXT: lw s0, 28(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s1, 24(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s2, 20(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s3, 16(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s4, 12(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s5, 8(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s6, 4(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s7, 0(sp) # 4-byte Folded Reload
+; RV32M-NEXT: addi sp, sp, 32
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_i128:
+; RV64M: # %bb.0:
+; RV64M-NEXT: mulhu a4, a0, a2
+; RV64M-NEXT: mul a5, a1, a2
+; RV64M-NEXT: mul a6, a3, a0
+; RV64M-NEXT: mulhsu a2, a1, a2
+; RV64M-NEXT: mulhsu a0, a3, a0
+; RV64M-NEXT: mulh a7, a1, a3
+; RV64M-NEXT: add a4, a5, a4
+; RV64M-NEXT: mul a1, a1, a3
+; RV64M-NEXT: add a3, a6, a4
+; RV64M-NEXT: sltu a4, a4, a5
+; RV64M-NEXT: sltu a3, a3, a6
+; RV64M-NEXT: add a2, a2, a4
+; RV64M-NEXT: add a0, a0, a3
+; RV64M-NEXT: add a3, a2, a0
+; RV64M-NEXT: srai a4, a2, 63
+; RV64M-NEXT: srai a0, a0, 63
+; RV64M-NEXT: sltu a2, a3, a2
+; RV64M-NEXT: add a0, a4, a0
+; RV64M-NEXT: add a2, a0, a2
+; RV64M-NEXT: add a0, a1, a3
+; RV64M-NEXT: add a2, a7, a2
+; RV64M-NEXT: sltu a1, a0, a1
+; RV64M-NEXT: add a1, a2, a1
+; RV64M-NEXT: ret
+ %r = call i128 @llvm.smulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @smulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; RV32I-LABEL: smulh_v8i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb s6, 16(a1)
+; RV32I-NEXT: lb s5, 20(a1)
+; RV32I-NEXT: lb a3, 24(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 28(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb s7, 16(a2)
+; RV32I-NEXT: lb s8, 20(a2)
+; RV32I-NEXT: lb a3, 24(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 28(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 0(a1)
+; RV32I-NEXT: lb s9, 4(a1)
+; RV32I-NEXT: lb s10, 8(a1)
+; RV32I-NEXT: lb s11, 12(a1)
+; RV32I-NEXT: lb a1, 0(a2)
+; RV32I-NEXT: lb s1, 4(a2)
+; RV32I-NEXT: lb s2, 8(a2)
+; RV32I-NEXT: lb s3, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 8
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 8
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 8
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 8
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 8
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s5, a0, 8
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 8
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sb s6, 4(s0)
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sb s5, 5(s0)
+; RV32I-NEXT: sb s7, 6(s0)
+; RV32I-NEXT: sb a0, 7(s0)
+; RV32I-NEXT: sb s4, 0(s0)
+; RV32I-NEXT: sb s1, 1(s0)
+; RV32I-NEXT: sb s2, 2(s0)
+; RV32I-NEXT: sb s3, 3(s0)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v8i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -144
+; RV64I-NEXT: sd ra, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb s6, 32(a1)
+; RV64I-NEXT: lb s5, 40(a1)
+; RV64I-NEXT: lb a3, 48(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 56(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb s7, 32(a2)
+; RV64I-NEXT: lb s8, 40(a2)
+; RV64I-NEXT: lb a3, 48(a2)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 56(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 0(a1)
+; RV64I-NEXT: lb s9, 8(a1)
+; RV64I-NEXT: lb s10, 16(a1)
+; RV64I-NEXT: lb s11, 24(a1)
+; RV64I-NEXT: lb a1, 0(a2)
+; RV64I-NEXT: lb s1, 8(a2)
+; RV64I-NEXT: lb s2, 16(a2)
+; RV64I-NEXT: lb s3, 24(a2)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 8
+; RV64I-NEXT: mv a0, s9
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 8
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 8
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 8
+; RV64I-NEXT: mv a0, s6
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 8
+; RV64I-NEXT: mv a0, s5
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 8
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 8
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sb s6, 4(s0)
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sb s5, 5(s0)
+; RV64I-NEXT: sb s7, 6(s0)
+; RV64I-NEXT: sb a0, 7(s0)
+; RV64I-NEXT: sb s4, 0(s0)
+; RV64I-NEXT: sb s1, 1(s0)
+; RV64I-NEXT: sb s2, 2(s0)
+; RV64I-NEXT: sb s3, 3(s0)
+; RV64I-NEXT: ld ra, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 144
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v8i8:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -16
+; RV32IM-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lb a3, 16(a2)
+; RV32IM-NEXT: lb a4, 20(a2)
+; RV32IM-NEXT: lb a5, 24(a2)
+; RV32IM-NEXT: lb a6, 28(a2)
+; RV32IM-NEXT: lb a7, 0(a2)
+; RV32IM-NEXT: lb t0, 4(a2)
+; RV32IM-NEXT: lb t1, 8(a2)
+; RV32IM-NEXT: lb a2, 12(a2)
+; RV32IM-NEXT: lb t2, 0(a1)
+; RV32IM-NEXT: lb t3, 4(a1)
+; RV32IM-NEXT: lb t4, 8(a1)
+; RV32IM-NEXT: lb t5, 12(a1)
+; RV32IM-NEXT: lb t6, 16(a1)
+; RV32IM-NEXT: lb s0, 20(a1)
+; RV32IM-NEXT: lb s1, 24(a1)
+; RV32IM-NEXT: lb a1, 28(a1)
+; RV32IM-NEXT: mul a7, t2, a7
+; RV32IM-NEXT: mul t0, t3, t0
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: mul a2, t5, a2
+; RV32IM-NEXT: mul a3, t6, a3
+; RV32IM-NEXT: mul a4, s0, a4
+; RV32IM-NEXT: mul a5, s1, a5
+; RV32IM-NEXT: mul a1, a1, a6
+; RV32IM-NEXT: srli a6, a7, 8
+; RV32IM-NEXT: srli a7, t0, 8
+; RV32IM-NEXT: srli t0, t1, 8
+; RV32IM-NEXT: srli a2, a2, 8
+; RV32IM-NEXT: srli a3, a3, 8
+; RV32IM-NEXT: srli a4, a4, 8
+; RV32IM-NEXT: srli a5, a5, 8
+; RV32IM-NEXT: srli a1, a1, 8
+; RV32IM-NEXT: sb a3, 4(a0)
+; RV32IM-NEXT: sb a4, 5(a0)
+; RV32IM-NEXT: sb a5, 6(a0)
+; RV32IM-NEXT: sb a1, 7(a0)
+; RV32IM-NEXT: sb a6, 0(a0)
+; RV32IM-NEXT: sb a7, 1(a0)
+; RV32IM-NEXT: sb t0, 2(a0)
+; RV32IM-NEXT: sb a2, 3(a0)
+; RV32IM-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 16
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v8i8:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -16
+; RV64IM-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lb a3, 32(a2)
+; RV64IM-NEXT: lb a4, 40(a2)
+; RV64IM-NEXT: lb a5, 48(a2)
+; RV64IM-NEXT: lb a6, 56(a2)
+; RV64IM-NEXT: lb a7, 0(a2)
+; RV64IM-NEXT: lb t0, 8(a2)
+; RV64IM-NEXT: lb t1, 16(a2)
+; RV64IM-NEXT: lb a2, 24(a2)
+; RV64IM-NEXT: lb t2, 0(a1)
+; RV64IM-NEXT: lb t3, 8(a1)
+; RV64IM-NEXT: lb t4, 16(a1)
+; RV64IM-NEXT: lb t5, 24(a1)
+; RV64IM-NEXT: lb t6, 32(a1)
+; RV64IM-NEXT: lb s0, 40(a1)
+; RV64IM-NEXT: lb s1, 48(a1)
+; RV64IM-NEXT: lb a1, 56(a1)
+; RV64IM-NEXT: mul a7, t2, a7
+; RV64IM-NEXT: mul t0, t3, t0
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: mul a2, t5, a2
+; RV64IM-NEXT: mul a3, t6, a3
+; RV64IM-NEXT: mul a4, s0, a4
+; RV64IM-NEXT: mul a5, s1, a5
+; RV64IM-NEXT: mul a1, a1, a6
+; RV64IM-NEXT: srli a6, a7, 8
+; RV64IM-NEXT: srli a7, t0, 8
+; RV64IM-NEXT: srli t0, t1, 8
+; RV64IM-NEXT: srli a2, a2, 8
+; RV64IM-NEXT: srli a3, a3, 8
+; RV64IM-NEXT: srli a4, a4, 8
+; RV64IM-NEXT: srli a5, a5, 8
+; RV64IM-NEXT: srli a1, a1, 8
+; RV64IM-NEXT: sb a3, 4(a0)
+; RV64IM-NEXT: sb a4, 5(a0)
+; RV64IM-NEXT: sb a5, 6(a0)
+; RV64IM-NEXT: sb a1, 7(a0)
+; RV64IM-NEXT: sb a6, 0(a0)
+; RV64IM-NEXT: sb a7, 1(a0)
+; RV64IM-NEXT: sb t0, 2(a0)
+; RV64IM-NEXT: sb a2, 3(a0)
+; RV64IM-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 16
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v8i8:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <8 x i8> @llvm.smulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @smulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; RV32I-LABEL: smulh_v4i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh a3, 0(a1)
+; RV32I-NEXT: lh s0, 4(a1)
+; RV32I-NEXT: lh s1, 8(a1)
+; RV32I-NEXT: lh s2, 12(a1)
+; RV32I-NEXT: lh a1, 0(a2)
+; RV32I-NEXT: lh s3, 4(a2)
+; RV32I-NEXT: lh s4, 8(a2)
+; RV32I-NEXT: lh s5, 12(a2)
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 16
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s0, a0, 16
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 16
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sh s7, 0(s6)
+; RV32I-NEXT: srli a0, a0, 16
+; RV32I-NEXT: sh s0, 2(s6)
+; RV32I-NEXT: sh s1, 4(s6)
+; RV32I-NEXT: sh a0, 6(s6)
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v4i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh a3, 0(a1)
+; RV64I-NEXT: lh s0, 8(a1)
+; RV64I-NEXT: lh s1, 16(a1)
+; RV64I-NEXT: lh s2, 24(a1)
+; RV64I-NEXT: lh a1, 0(a2)
+; RV64I-NEXT: lh s3, 8(a2)
+; RV64I-NEXT: lh s4, 16(a2)
+; RV64I-NEXT: lh s5, 24(a2)
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 16
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 16
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 16
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sh s7, 0(s6)
+; RV64I-NEXT: srli a0, a0, 16
+; RV64I-NEXT: sh s0, 2(s6)
+; RV64I-NEXT: sh s1, 4(s6)
+; RV64I-NEXT: sh a0, 6(s6)
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v4i16:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lh a3, 0(a2)
+; RV32IM-NEXT: lh a4, 4(a2)
+; RV32IM-NEXT: lh a5, 8(a2)
+; RV32IM-NEXT: lh a2, 12(a2)
+; RV32IM-NEXT: lh a6, 0(a1)
+; RV32IM-NEXT: lh a7, 4(a1)
+; RV32IM-NEXT: lh t0, 8(a1)
+; RV32IM-NEXT: lh a1, 12(a1)
+; RV32IM-NEXT: mul a3, a6, a3
+; RV32IM-NEXT: mul a4, a7, a4
+; RV32IM-NEXT: mul a5, t0, a5
+; RV32IM-NEXT: mul a1, a1, a2
+; RV32IM-NEXT: srli a3, a3, 16
+; RV32IM-NEXT: srli a4, a4, 16
+; RV32IM-NEXT: srli a5, a5, 16
+; RV32IM-NEXT: srli a1, a1, 16
+; RV32IM-NEXT: sh a3, 0(a0)
+; RV32IM-NEXT: sh a4, 2(a0)
+; RV32IM-NEXT: sh a5, 4(a0)
+; RV32IM-NEXT: sh a1, 6(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v4i16:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lh a3, 0(a2)
+; RV64IM-NEXT: lh a4, 8(a2)
+; RV64IM-NEXT: lh a5, 16(a2)
+; RV64IM-NEXT: lh a2, 24(a2)
+; RV64IM-NEXT: lh a6, 0(a1)
+; RV64IM-NEXT: lh a7, 8(a1)
+; RV64IM-NEXT: lh t0, 16(a1)
+; RV64IM-NEXT: lh a1, 24(a1)
+; RV64IM-NEXT: mul a3, a6, a3
+; RV64IM-NEXT: mul a4, a7, a4
+; RV64IM-NEXT: mul a5, t0, a5
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 16
+; RV64IM-NEXT: srli a4, a4, 16
+; RV64IM-NEXT: srli a5, a5, 16
+; RV64IM-NEXT: srli a1, a1, 16
+; RV64IM-NEXT: sh a3, 0(a0)
+; RV64IM-NEXT: sh a4, 2(a0)
+; RV64IM-NEXT: sh a5, 4(a0)
+; RV64IM-NEXT: sh a1, 6(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v4i16:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <4 x i16> @llvm.smulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @smulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; RV32I-LABEL: smulh_v2i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: srai a1, a0, 31
+; RV32I-NEXT: srai a3, a2, 31
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: srai a1, s1, 31
+; RV32I-NEXT: srai a3, s0, 31
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v2i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sext.w s0, a1
+; RV64I-NEXT: sext.w s1, a3
+; RV64I-NEXT: sext.w a0, a0
+; RV64I-NEXT: sext.w a1, a2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srai s2, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: srai a1, a1, 32
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v2i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: mulh a0, a0, a2
+; RV32IM-NEXT: mulh a1, a1, a3
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v2i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: sext.w a3, a3
+; RV64IM-NEXT: sext.w a1, a1
+; RV64IM-NEXT: sext.w a2, a2
+; RV64IM-NEXT: sext.w a0, a0
+; RV64IM-NEXT: mul a0, a0, a2
+; RV64IM-NEXT: mul a1, a1, a3
+; RV64IM-NEXT: srai a0, a0, 32
+; RV64IM-NEXT: srai a1, a1, 32
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v2i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <2 x i32> @llvm.smulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @smulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; RV32I-LABEL: smulh_v1i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: srai s4, a1, 31
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, s5, a1
+; RV32I-NEXT: sltu a0, s7, s5
+; RV32I-NEXT: add s6, s6, a0
+; RV32I-NEXT: srai s1, s0, 31
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, a0, s7
+; RV32I-NEXT: sltu a0, s7, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s6, a0
+; RV32I-NEXT: srai a1, s6, 31
+; RV32I-NEXT: srai a0, a0, 31
+; RV32I-NEXT: sltu a2, s3, s6
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s5, a0, a2
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: add a1, a1, s5
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a1, a1, a0
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v1i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a2, a1
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: srai a3, a2, 63
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v1i64:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: mulhu a4, a0, a2
+; RV32IM-NEXT: mul a5, a1, a2
+; RV32IM-NEXT: mul a6, a3, a0
+; RV32IM-NEXT: mulhsu a2, a1, a2
+; RV32IM-NEXT: mulhsu a0, a3, a0
+; RV32IM-NEXT: mulh a7, a1, a3
+; RV32IM-NEXT: add a4, a5, a4
+; RV32IM-NEXT: mul a1, a1, a3
+; RV32IM-NEXT: add a3, a6, a4
+; RV32IM-NEXT: sltu a4, a4, a5
+; RV32IM-NEXT: sltu a3, a3, a6
+; RV32IM-NEXT: add a2, a2, a4
+; RV32IM-NEXT: add a0, a0, a3
+; RV32IM-NEXT: add a3, a2, a0
+; RV32IM-NEXT: srai a4, a2, 31
+; RV32IM-NEXT: srai a0, a0, 31
+; RV32IM-NEXT: sltu a2, a3, a2
+; RV32IM-NEXT: add a0, a4, a0
+; RV32IM-NEXT: add a2, a0, a2
+; RV32IM-NEXT: add a0, a1, a3
+; RV32IM-NEXT: add a2, a7, a2
+; RV32IM-NEXT: sltu a1, a0, a1
+; RV32IM-NEXT: add a1, a2, a1
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v1i64:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: mulh a0, a0, a1
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v1i64:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <1 x i64> @llvm.smulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @smulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; RV32I-LABEL: smulh_v16i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -144
+; RV32I-NEXT: sw ra, 140(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 136(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 132(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 128(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 120(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 116(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 112(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 104(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 100(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 96(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 48(a1)
+; RV32I-NEXT: sw a3, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 52(a1)
+; RV32I-NEXT: sw a3, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 56(a1)
+; RV32I-NEXT: sw a3, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 60(a1)
+; RV32I-NEXT: sw a3, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 48(a2)
+; RV32I-NEXT: sw a3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 52(a2)
+; RV32I-NEXT: sw a3, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 56(a2)
+; RV32I-NEXT: sw a3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 60(a2)
+; RV32I-NEXT: sw a3, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 32(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 36(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 40(a1)
+; RV32I-NEXT: sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 44(a1)
+; RV32I-NEXT: sw a3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 32(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 36(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 40(a2)
+; RV32I-NEXT: sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 44(a2)
+; RV32I-NEXT: sw a3, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb s10, 16(a1)
+; RV32I-NEXT: lb a3, 20(a1)
+; RV32I-NEXT: sw a3, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 24(a1)
+; RV32I-NEXT: sw a3, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 28(a1)
+; RV32I-NEXT: sw a3, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lb a3, 0(a1)
+; RV32I-NEXT: lb s11, 4(a1)
+; RV32I-NEXT: lb s0, 8(a1)
+; RV32I-NEXT: lb s1, 12(a1)
+; RV32I-NEXT: lb s2, 16(a2)
+; RV32I-NEXT: lb s3, 20(a2)
+; RV32I-NEXT: lb s4, 24(a2)
+; RV32I-NEXT: lb s9, 28(a2)
+; RV32I-NEXT: lb a1, 0(a2)
+; RV32I-NEXT: lb s6, 4(a2)
+; RV32I-NEXT: lb s7, 8(a2)
+; RV32I-NEXT: lb s8, 12(a2)
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s6
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 8
+; RV32I-NEXT: lw a0, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 8
+; RV32I-NEXT: lw a0, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 8
+; RV32I-NEXT: lw a0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s9
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 8
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s8, a0, 8
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s9, a0, 8
+; RV32I-NEXT: lw a0, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s10, a0, 8
+; RV32I-NEXT: lw a0, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s11, a0, 8
+; RV32I-NEXT: lw a0, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 8
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s0, a0, 8
+; RV32I-NEXT: lw a0, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 8
+; RV32I-NEXT: lw a0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sb s6, 12(s5)
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sb s0, 13(s5)
+; RV32I-NEXT: sb s1, 14(s5)
+; RV32I-NEXT: sb a0, 15(s5)
+; RV32I-NEXT: sb s8, 8(s5)
+; RV32I-NEXT: sb s9, 9(s5)
+; RV32I-NEXT: sb s10, 10(s5)
+; RV32I-NEXT: sb s11, 11(s5)
+; RV32I-NEXT: sb s2, 4(s5)
+; RV32I-NEXT: sb s3, 5(s5)
+; RV32I-NEXT: sb s4, 6(s5)
+; RV32I-NEXT: sb s7, 7(s5)
+; RV32I-NEXT: lw a0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 0(s5)
+; RV32I-NEXT: lw a0, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 1(s5)
+; RV32I-NEXT: lw a0, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 2(s5)
+; RV32I-NEXT: lw a0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 3(s5)
+; RV32I-NEXT: lw ra, 140(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 136(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 132(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 128(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 124(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 116(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 112(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 108(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 100(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 96(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 144
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v16i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -288
+; RV64I-NEXT: sd ra, 280(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 272(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 264(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 256(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 248(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 240(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 232(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 224(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 216(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 208(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 200(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 192(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 184(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 96(a1)
+; RV64I-NEXT: sd a3, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 104(a1)
+; RV64I-NEXT: sd a3, 144(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 112(a1)
+; RV64I-NEXT: sd a3, 160(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 120(a1)
+; RV64I-NEXT: sd a3, 176(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 96(a2)
+; RV64I-NEXT: sd a3, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 104(a2)
+; RV64I-NEXT: sd a3, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 112(a2)
+; RV64I-NEXT: sd a3, 152(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 120(a2)
+; RV64I-NEXT: sd a3, 168(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 64(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 72(a1)
+; RV64I-NEXT: sd a3, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 80(a1)
+; RV64I-NEXT: sd a3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 88(a1)
+; RV64I-NEXT: sd a3, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 64(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 72(a2)
+; RV64I-NEXT: sd a3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 80(a2)
+; RV64I-NEXT: sd a3, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 88(a2)
+; RV64I-NEXT: sd a3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb s10, 32(a1)
+; RV64I-NEXT: lb a3, 40(a1)
+; RV64I-NEXT: sd a3, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 48(a1)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 56(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lb a3, 0(a1)
+; RV64I-NEXT: lb s11, 8(a1)
+; RV64I-NEXT: lb s0, 16(a1)
+; RV64I-NEXT: lb s1, 24(a1)
+; RV64I-NEXT: lb s2, 32(a2)
+; RV64I-NEXT: lb s3, 40(a2)
+; RV64I-NEXT: lb s4, 48(a2)
+; RV64I-NEXT: lb s9, 56(a2)
+; RV64I-NEXT: lb a1, 0(a2)
+; RV64I-NEXT: lb s6, 8(a2)
+; RV64I-NEXT: lb s7, 16(a2)
+; RV64I-NEXT: lb s8, 24(a2)
+; RV64I-NEXT: mv s5, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s6
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 8
+; RV64I-NEXT: ld a0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 8
+; RV64I-NEXT: ld a0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 8
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s9
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 8
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s8, a0, 8
+; RV64I-NEXT: ld a0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s9, a0, 8
+; RV64I-NEXT: ld a0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s10, a0, 8
+; RV64I-NEXT: ld a0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s11, a0, 8
+; RV64I-NEXT: ld a0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 8
+; RV64I-NEXT: ld a0, 144(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 8
+; RV64I-NEXT: ld a0, 160(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 152(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 8
+; RV64I-NEXT: ld a0, 176(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 168(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sb s6, 12(s5)
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sb s0, 13(s5)
+; RV64I-NEXT: sb s1, 14(s5)
+; RV64I-NEXT: sb a0, 15(s5)
+; RV64I-NEXT: sb s8, 8(s5)
+; RV64I-NEXT: sb s9, 9(s5)
+; RV64I-NEXT: sb s10, 10(s5)
+; RV64I-NEXT: sb s11, 11(s5)
+; RV64I-NEXT: sb s2, 4(s5)
+; RV64I-NEXT: sb s3, 5(s5)
+; RV64I-NEXT: sb s4, 6(s5)
+; RV64I-NEXT: sb s7, 7(s5)
+; RV64I-NEXT: ld a0, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 0(s5)
+; RV64I-NEXT: ld a0, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 1(s5)
+; RV64I-NEXT: ld a0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 2(s5)
+; RV64I-NEXT: ld a0, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 3(s5)
+; RV64I-NEXT: ld ra, 280(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 272(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 264(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 256(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 248(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 240(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 232(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 224(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 216(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 208(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 200(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 192(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 288
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v16i8:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -48
+; RV32IM-NEXT: sw s0, 44(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 40(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s3, 32(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s5, 24(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s6, 20(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s7, 16(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s8, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s9, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s10, 4(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s11, 0(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lb t3, 48(a2)
+; RV32IM-NEXT: lb t2, 52(a2)
+; RV32IM-NEXT: lb t1, 56(a2)
+; RV32IM-NEXT: lb a4, 60(a2)
+; RV32IM-NEXT: lb t6, 48(a1)
+; RV32IM-NEXT: lb t5, 52(a1)
+; RV32IM-NEXT: lb t4, 56(a1)
+; RV32IM-NEXT: lb t0, 60(a1)
+; RV32IM-NEXT: lb s2, 16(a2)
+; RV32IM-NEXT: lb s3, 20(a2)
+; RV32IM-NEXT: lb s1, 24(a2)
+; RV32IM-NEXT: lb s0, 28(a2)
+; RV32IM-NEXT: lb a3, 0(a2)
+; RV32IM-NEXT: lb a5, 0(a1)
+; RV32IM-NEXT: lb a6, 4(a1)
+; RV32IM-NEXT: lb a7, 4(a2)
+; RV32IM-NEXT: lb s4, 8(a2)
+; RV32IM-NEXT: lb s5, 12(a2)
+; RV32IM-NEXT: lb s6, 8(a1)
+; RV32IM-NEXT: lb s7, 12(a1)
+; RV32IM-NEXT: lb s8, 16(a1)
+; RV32IM-NEXT: mul a3, a5, a3
+; RV32IM-NEXT: mul a5, a6, a7
+; RV32IM-NEXT: lb s9, 20(a1)
+; RV32IM-NEXT: lb s10, 24(a1)
+; RV32IM-NEXT: lb s11, 28(a1)
+; RV32IM-NEXT: mul a6, s6, s4
+; RV32IM-NEXT: mul a7, s7, s5
+; RV32IM-NEXT: mul s2, s8, s2
+; RV32IM-NEXT: lb s4, 32(a2)
+; RV32IM-NEXT: lb s5, 36(a2)
+; RV32IM-NEXT: lb s6, 40(a2)
+; RV32IM-NEXT: lb a2, 44(a2)
+; RV32IM-NEXT: mul t3, t6, t3
+; RV32IM-NEXT: mul t2, t5, t2
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: lb t4, 32(a1)
+; RV32IM-NEXT: lb t5, 36(a1)
+; RV32IM-NEXT: lb t6, 40(a1)
+; RV32IM-NEXT: lb a1, 44(a1)
+; RV32IM-NEXT: mul s3, s9, s3
+; RV32IM-NEXT: mul a4, t0, a4
+; RV32IM-NEXT: mul t0, s10, s1
+; RV32IM-NEXT: mul s0, s11, s0
+; RV32IM-NEXT: mul t4, t4, s4
+; RV32IM-NEXT: mul t5, t5, s5
+; RV32IM-NEXT: mul t6, t6, s6
+; RV32IM-NEXT: mul a1, a1, a2
+; RV32IM-NEXT: srli a2, t3, 8
+; RV32IM-NEXT: srli t2, t2, 8
+; RV32IM-NEXT: sb a2, 12(a0)
+; RV32IM-NEXT: srli a2, t1, 8
+; RV32IM-NEXT: sb t2, 13(a0)
+; RV32IM-NEXT: srli a4, a4, 8
+; RV32IM-NEXT: sb a2, 14(a0)
+; RV32IM-NEXT: sb a4, 15(a0)
+; RV32IM-NEXT: srli a2, t4, 8
+; RV32IM-NEXT: srli a4, t5, 8
+; RV32IM-NEXT: sb a2, 8(a0)
+; RV32IM-NEXT: srli a2, t6, 8
+; RV32IM-NEXT: sb a4, 9(a0)
+; RV32IM-NEXT: srli a1, a1, 8
+; RV32IM-NEXT: sb a2, 10(a0)
+; RV32IM-NEXT: sb a1, 11(a0)
+; RV32IM-NEXT: srli a1, s2, 8
+; RV32IM-NEXT: srli a2, s3, 8
+; RV32IM-NEXT: sb a1, 4(a0)
+; RV32IM-NEXT: srli a1, t0, 8
+; RV32IM-NEXT: sb a2, 5(a0)
+; RV32IM-NEXT: srli s0, s0, 8
+; RV32IM-NEXT: sb a1, 6(a0)
+; RV32IM-NEXT: sb s0, 7(a0)
+; RV32IM-NEXT: srli a3, a3, 8
+; RV32IM-NEXT: srli a5, a5, 8
+; RV32IM-NEXT: sb a3, 0(a0)
+; RV32IM-NEXT: srli a1, a6, 8
+; RV32IM-NEXT: sb a5, 1(a0)
+; RV32IM-NEXT: srli a2, a7, 8
+; RV32IM-NEXT: sb a1, 2(a0)
+; RV32IM-NEXT: sb a2, 3(a0)
+; RV32IM-NEXT: lw s0, 44(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s2, 36(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s5, 24(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s7, 16(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s8, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s9, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s10, 4(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s11, 0(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 48
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v16i8:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -96
+; RV64IM-NEXT: sd s0, 88(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 80(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s2, 72(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s3, 64(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s4, 56(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s5, 48(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s6, 40(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s7, 32(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s8, 24(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s9, 16(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s10, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s11, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lb t3, 96(a2)
+; RV64IM-NEXT: lb t2, 104(a2)
+; RV64IM-NEXT: lb t1, 112(a2)
+; RV64IM-NEXT: lb a4, 120(a2)
+; RV64IM-NEXT: lb t6, 96(a1)
+; RV64IM-NEXT: lb t5, 104(a1)
+; RV64IM-NEXT: lb t4, 112(a1)
+; RV64IM-NEXT: lb t0, 120(a1)
+; RV64IM-NEXT: lb s2, 32(a2)
+; RV64IM-NEXT: lb s3, 40(a2)
+; RV64IM-NEXT: lb s1, 48(a2)
+; RV64IM-NEXT: lb s0, 56(a2)
+; RV64IM-NEXT: lb a3, 0(a2)
+; RV64IM-NEXT: lb a5, 0(a1)
+; RV64IM-NEXT: lb a6, 8(a1)
+; RV64IM-NEXT: lb a7, 8(a2)
+; RV64IM-NEXT: lb s4, 16(a2)
+; RV64IM-NEXT: lb s5, 24(a2)
+; RV64IM-NEXT: lb s6, 16(a1)
+; RV64IM-NEXT: lb s7, 24(a1)
+; RV64IM-NEXT: lb s8, 32(a1)
+; RV64IM-NEXT: mul a3, a5, a3
+; RV64IM-NEXT: mul a5, a6, a7
+; RV64IM-NEXT: lb s9, 40(a1)
+; RV64IM-NEXT: lb s10, 48(a1)
+; RV64IM-NEXT: lb s11, 56(a1)
+; RV64IM-NEXT: mul a6, s6, s4
+; RV64IM-NEXT: mul a7, s7, s5
+; RV64IM-NEXT: mul s2, s8, s2
+; RV64IM-NEXT: lb s4, 64(a2)
+; RV64IM-NEXT: lb s5, 72(a2)
+; RV64IM-NEXT: lb s6, 80(a2)
+; RV64IM-NEXT: lb a2, 88(a2)
+; RV64IM-NEXT: mul t3, t6, t3
+; RV64IM-NEXT: mul t2, t5, t2
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: lb t4, 64(a1)
+; RV64IM-NEXT: lb t5, 72(a1)
+; RV64IM-NEXT: lb t6, 80(a1)
+; RV64IM-NEXT: lb a1, 88(a1)
+; RV64IM-NEXT: mul s3, s9, s3
+; RV64IM-NEXT: mul a4, t0, a4
+; RV64IM-NEXT: mul t0, s10, s1
+; RV64IM-NEXT: mul s0, s11, s0
+; RV64IM-NEXT: mul t4, t4, s4
+; RV64IM-NEXT: mul t5, t5, s5
+; RV64IM-NEXT: mul t6, t6, s6
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a2, t3, 8
+; RV64IM-NEXT: srli t2, t2, 8
+; RV64IM-NEXT: sb a2, 12(a0)
+; RV64IM-NEXT: srli a2, t1, 8
+; RV64IM-NEXT: sb t2, 13(a0)
+; RV64IM-NEXT: srli a4, a4, 8
+; RV64IM-NEXT: sb a2, 14(a0)
+; RV64IM-NEXT: sb a4, 15(a0)
+; RV64IM-NEXT: srli a2, t4, 8
+; RV64IM-NEXT: srli a4, t5, 8
+; RV64IM-NEXT: sb a2, 8(a0)
+; RV64IM-NEXT: srli a2, t6, 8
+; RV64IM-NEXT: sb a4, 9(a0)
+; RV64IM-NEXT: srli a1, a1, 8
+; RV64IM-NEXT: sb a2, 10(a0)
+; RV64IM-NEXT: sb a1, 11(a0)
+; RV64IM-NEXT: srli a1, s2, 8
+; RV64IM-NEXT: srli a2, s3, 8
+; RV64IM-NEXT: sb a1, 4(a0)
+; RV64IM-NEXT: srli a1, t0, 8
+; RV64IM-NEXT: sb a2, 5(a0)
+; RV64IM-NEXT: srli s0, s0, 8
+; RV64IM-NEXT: sb a1, 6(a0)
+; RV64IM-NEXT: sb s0, 7(a0)
+; RV64IM-NEXT: srli a3, a3, 8
+; RV64IM-NEXT: srli a5, a5, 8
+; RV64IM-NEXT: sb a3, 0(a0)
+; RV64IM-NEXT: srli a1, a6, 8
+; RV64IM-NEXT: sb a5, 1(a0)
+; RV64IM-NEXT: srli a2, a7, 8
+; RV64IM-NEXT: sb a1, 2(a0)
+; RV64IM-NEXT: sb a2, 3(a0)
+; RV64IM-NEXT: ld s0, 88(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 80(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s2, 72(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s3, 64(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s4, 56(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s5, 48(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s6, 40(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s7, 32(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s8, 24(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s9, 16(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s10, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s11, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 96
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v16i8:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <16 x i8> @llvm.smulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @smulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; RV32I-LABEL: smulh_v8i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh s6, 16(a1)
+; RV32I-NEXT: lh s5, 20(a1)
+; RV32I-NEXT: lh a3, 24(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh a3, 28(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh s7, 16(a2)
+; RV32I-NEXT: lh s8, 20(a2)
+; RV32I-NEXT: lh a3, 24(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh a3, 28(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lh a3, 0(a1)
+; RV32I-NEXT: lh s9, 4(a1)
+; RV32I-NEXT: lh s10, 8(a1)
+; RV32I-NEXT: lh s11, 12(a1)
+; RV32I-NEXT: lh a1, 0(a2)
+; RV32I-NEXT: lh s1, 4(a2)
+; RV32I-NEXT: lh s2, 8(a2)
+; RV32I-NEXT: lh s3, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 16
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 16
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 16
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 16
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 16
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s5, a0, 16
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 16
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sh s6, 8(s0)
+; RV32I-NEXT: srli a0, a0, 16
+; RV32I-NEXT: sh s5, 10(s0)
+; RV32I-NEXT: sh s7, 12(s0)
+; RV32I-NEXT: sh a0, 14(s0)
+; RV32I-NEXT: sh s4, 0(s0)
+; RV32I-NEXT: sh s1, 2(s0)
+; RV32I-NEXT: sh s2, 4(s0)
+; RV32I-NEXT: sh s3, 6(s0)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v8i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -144
+; RV64I-NEXT: sd ra, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh s6, 32(a1)
+; RV64I-NEXT: lh s5, 40(a1)
+; RV64I-NEXT: lh a3, 48(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh a3, 56(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh s7, 32(a2)
+; RV64I-NEXT: lh s8, 40(a2)
+; RV64I-NEXT: lh a3, 48(a2)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh a3, 56(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lh a3, 0(a1)
+; RV64I-NEXT: lh s9, 8(a1)
+; RV64I-NEXT: lh s10, 16(a1)
+; RV64I-NEXT: lh s11, 24(a1)
+; RV64I-NEXT: lh a1, 0(a2)
+; RV64I-NEXT: lh s1, 8(a2)
+; RV64I-NEXT: lh s2, 16(a2)
+; RV64I-NEXT: lh s3, 24(a2)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 16
+; RV64I-NEXT: mv a0, s9
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 16
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 16
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 16
+; RV64I-NEXT: mv a0, s6
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 16
+; RV64I-NEXT: mv a0, s5
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 16
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 16
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sh s6, 8(s0)
+; RV64I-NEXT: srli a0, a0, 16
+; RV64I-NEXT: sh s5, 10(s0)
+; RV64I-NEXT: sh s7, 12(s0)
+; RV64I-NEXT: sh a0, 14(s0)
+; RV64I-NEXT: sh s4, 0(s0)
+; RV64I-NEXT: sh s1, 2(s0)
+; RV64I-NEXT: sh s2, 4(s0)
+; RV64I-NEXT: sh s3, 6(s0)
+; RV64I-NEXT: ld ra, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 144
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v8i16:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -16
+; RV32IM-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lh a3, 16(a2)
+; RV32IM-NEXT: lh a4, 20(a2)
+; RV32IM-NEXT: lh a5, 24(a2)
+; RV32IM-NEXT: lh a6, 28(a2)
+; RV32IM-NEXT: lh a7, 0(a2)
+; RV32IM-NEXT: lh t0, 4(a2)
+; RV32IM-NEXT: lh t1, 8(a2)
+; RV32IM-NEXT: lh a2, 12(a2)
+; RV32IM-NEXT: lh t2, 0(a1)
+; RV32IM-NEXT: lh t3, 4(a1)
+; RV32IM-NEXT: lh t4, 8(a1)
+; RV32IM-NEXT: lh t5, 12(a1)
+; RV32IM-NEXT: lh t6, 16(a1)
+; RV32IM-NEXT: lh s0, 20(a1)
+; RV32IM-NEXT: lh s1, 24(a1)
+; RV32IM-NEXT: lh a1, 28(a1)
+; RV32IM-NEXT: mul a7, t2, a7
+; RV32IM-NEXT: mul t0, t3, t0
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: mul a2, t5, a2
+; RV32IM-NEXT: mul a3, t6, a3
+; RV32IM-NEXT: mul a4, s0, a4
+; RV32IM-NEXT: mul a5, s1, a5
+; RV32IM-NEXT: mul a1, a1, a6
+; RV32IM-NEXT: srli a6, a7, 16
+; RV32IM-NEXT: srli a7, t0, 16
+; RV32IM-NEXT: srli t0, t1, 16
+; RV32IM-NEXT: srli a2, a2, 16
+; RV32IM-NEXT: srli a3, a3, 16
+; RV32IM-NEXT: srli a4, a4, 16
+; RV32IM-NEXT: srli a5, a5, 16
+; RV32IM-NEXT: srli a1, a1, 16
+; RV32IM-NEXT: sh a3, 8(a0)
+; RV32IM-NEXT: sh a4, 10(a0)
+; RV32IM-NEXT: sh a5, 12(a0)
+; RV32IM-NEXT: sh a1, 14(a0)
+; RV32IM-NEXT: sh a6, 0(a0)
+; RV32IM-NEXT: sh a7, 2(a0)
+; RV32IM-NEXT: sh t0, 4(a0)
+; RV32IM-NEXT: sh a2, 6(a0)
+; RV32IM-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 16
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v8i16:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -16
+; RV64IM-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lh a3, 32(a2)
+; RV64IM-NEXT: lh a4, 40(a2)
+; RV64IM-NEXT: lh a5, 48(a2)
+; RV64IM-NEXT: lh a6, 56(a2)
+; RV64IM-NEXT: lh a7, 0(a2)
+; RV64IM-NEXT: lh t0, 8(a2)
+; RV64IM-NEXT: lh t1, 16(a2)
+; RV64IM-NEXT: lh a2, 24(a2)
+; RV64IM-NEXT: lh t2, 0(a1)
+; RV64IM-NEXT: lh t3, 8(a1)
+; RV64IM-NEXT: lh t4, 16(a1)
+; RV64IM-NEXT: lh t5, 24(a1)
+; RV64IM-NEXT: lh t6, 32(a1)
+; RV64IM-NEXT: lh s0, 40(a1)
+; RV64IM-NEXT: lh s1, 48(a1)
+; RV64IM-NEXT: lh a1, 56(a1)
+; RV64IM-NEXT: mul a7, t2, a7
+; RV64IM-NEXT: mul t0, t3, t0
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: mul a2, t5, a2
+; RV64IM-NEXT: mul a3, t6, a3
+; RV64IM-NEXT: mul a4, s0, a4
+; RV64IM-NEXT: mul a5, s1, a5
+; RV64IM-NEXT: mul a1, a1, a6
+; RV64IM-NEXT: srli a6, a7, 16
+; RV64IM-NEXT: srli a7, t0, 16
+; RV64IM-NEXT: srli t0, t1, 16
+; RV64IM-NEXT: srli a2, a2, 16
+; RV64IM-NEXT: srli a3, a3, 16
+; RV64IM-NEXT: srli a4, a4, 16
+; RV64IM-NEXT: srli a5, a5, 16
+; RV64IM-NEXT: srli a1, a1, 16
+; RV64IM-NEXT: sh a3, 8(a0)
+; RV64IM-NEXT: sh a4, 10(a0)
+; RV64IM-NEXT: sh a5, 12(a0)
+; RV64IM-NEXT: sh a1, 14(a0)
+; RV64IM-NEXT: sh a6, 0(a0)
+; RV64IM-NEXT: sh a7, 2(a0)
+; RV64IM-NEXT: sh t0, 4(a0)
+; RV64IM-NEXT: sh a2, 6(a0)
+; RV64IM-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 16
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v8i16:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <8 x i16> @llvm.smulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @smulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; RV32I-LABEL: smulh_v4i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a4, 0(a2)
+; RV32I-NEXT: lw s0, 4(a2)
+; RV32I-NEXT: lw s1, 8(a2)
+; RV32I-NEXT: lw s2, 12(a2)
+; RV32I-NEXT: lw a2, 0(a1)
+; RV32I-NEXT: lw s3, 4(a1)
+; RV32I-NEXT: lw s4, 8(a1)
+; RV32I-NEXT: lw s5, 12(a1)
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: srai a1, a2, 31
+; RV32I-NEXT: srai a3, a4, 31
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: mv a2, a4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: srai a1, s3, 31
+; RV32I-NEXT: srai a3, s0, 31
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: srai a1, s4, 31
+; RV32I-NEXT: srai a3, s1, 31
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: srai a1, s5, 31
+; RV32I-NEXT: srai a3, s2, 31
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw s7, 0(s6)
+; RV32I-NEXT: sw s0, 4(s6)
+; RV32I-NEXT: sw s1, 8(s6)
+; RV32I-NEXT: sw a1, 12(s6)
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v4i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lw a3, 0(a1)
+; RV64I-NEXT: lw s0, 8(a1)
+; RV64I-NEXT: lw s1, 16(a1)
+; RV64I-NEXT: lw s2, 24(a1)
+; RV64I-NEXT: lw a1, 0(a2)
+; RV64I-NEXT: lw s3, 8(a2)
+; RV64I-NEXT: lw s4, 16(a2)
+; RV64I-NEXT: lw s5, 24(a2)
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 32
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 32
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sw s7, 0(s6)
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: sw s0, 4(s6)
+; RV64I-NEXT: sw s1, 8(s6)
+; RV64I-NEXT: sw a0, 12(s6)
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v4i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a2)
+; RV32IM-NEXT: lw a4, 4(a2)
+; RV32IM-NEXT: lw a5, 8(a2)
+; RV32IM-NEXT: lw a2, 12(a2)
+; RV32IM-NEXT: lw a6, 0(a1)
+; RV32IM-NEXT: lw a7, 4(a1)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: mulh a3, a6, a3
+; RV32IM-NEXT: mulh a4, a7, a4
+; RV32IM-NEXT: mulh a5, t0, a5
+; RV32IM-NEXT: mulh a1, a1, a2
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a4, 4(a0)
+; RV32IM-NEXT: sw a5, 8(a0)
+; RV32IM-NEXT: sw a1, 12(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v4i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lw a3, 0(a2)
+; RV64IM-NEXT: lw a4, 8(a2)
+; RV64IM-NEXT: lw a5, 16(a2)
+; RV64IM-NEXT: lw a2, 24(a2)
+; RV64IM-NEXT: lw a6, 0(a1)
+; RV64IM-NEXT: lw a7, 8(a1)
+; RV64IM-NEXT: lw t0, 16(a1)
+; RV64IM-NEXT: lw a1, 24(a1)
+; RV64IM-NEXT: mul a3, a6, a3
+; RV64IM-NEXT: mul a4, a7, a4
+; RV64IM-NEXT: mul a5, t0, a5
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 32
+; RV64IM-NEXT: srli a4, a4, 32
+; RV64IM-NEXT: srli a5, a5, 32
+; RV64IM-NEXT: srli a1, a1, 32
+; RV64IM-NEXT: sw a3, 0(a0)
+; RV64IM-NEXT: sw a4, 4(a0)
+; RV64IM-NEXT: sw a5, 8(a0)
+; RV64IM-NEXT: sw a1, 12(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v4i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <4 x i32> @llvm.smulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @smulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; RV32I-LABEL: smulh_v2i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -64
+; RV32I-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s5, 0(a2)
+; RV32I-NEXT: lw s6, 4(a2)
+; RV32I-NEXT: lw s4, 8(a2)
+; RV32I-NEXT: lw s1, 12(a2)
+; RV32I-NEXT: lw s7, 0(a1)
+; RV32I-NEXT: lw s8, 4(a1)
+; RV32I-NEXT: lw a2, 8(a1)
+; RV32I-NEXT: sw a2, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s2, 12(a1)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: srai s9, s8, 31
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: mv a1, s9
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s10, a0
+; RV32I-NEXT: mv s11, a1
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, s10, a1
+; RV32I-NEXT: sltu a0, s3, s10
+; RV32I-NEXT: add s11, s11, a0
+; RV32I-NEXT: srai s5, s6, 31
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: mv a3, s5
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s11, a0
+; RV32I-NEXT: srai a1, s11, 31
+; RV32I-NEXT: srai a0, a0, 31
+; RV32I-NEXT: sltu a2, s3, s11
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s7, a0, a2
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: mv a1, s9
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: mv a3, s5
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s8, a0, s3
+; RV32I-NEXT: add a1, a1, s7
+; RV32I-NEXT: sltu s9, s8, a0
+; RV32I-NEXT: add s9, a1, s9
+; RV32I-NEXT: srai s5, s2, 31
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: lw s10, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, s6, a1
+; RV32I-NEXT: sltu a0, s3, s6
+; RV32I-NEXT: add s7, s7, a0
+; RV32I-NEXT: srai s4, s1, 31
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: mv a3, s4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s7, a0
+; RV32I-NEXT: srai a1, s7, 31
+; RV32I-NEXT: srai a0, a0, 31
+; RV32I-NEXT: sltu a2, s3, s7
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s6, a0, a2
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: mv a3, s4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a1, a1, s6
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw s8, 0(s0)
+; RV32I-NEXT: sw s9, 4(s0)
+; RV32I-NEXT: sw s3, 8(s0)
+; RV32I-NEXT: sw a0, 12(s0)
+; RV32I-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 64
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v2i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a3
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: srai a3, a2, 63
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: srai a1, s1, 63
+; RV64I-NEXT: srai a3, s0, 63
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v2i64:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a1)
+; RV32IM-NEXT: lw a4, 4(a1)
+; RV32IM-NEXT: lw a5, 0(a2)
+; RV32IM-NEXT: lw a6, 4(a2)
+; RV32IM-NEXT: lw a7, 8(a2)
+; RV32IM-NEXT: lw a2, 12(a2)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: mulhu t1, a3, a5
+; RV32IM-NEXT: mul t2, a4, a5
+; RV32IM-NEXT: mul t3, a6, a3
+; RV32IM-NEXT: mulhsu a5, a4, a5
+; RV32IM-NEXT: mulhsu a3, a6, a3
+; RV32IM-NEXT: mulh t4, a4, a6
+; RV32IM-NEXT: mulhu t5, t0, a7
+; RV32IM-NEXT: mul t6, a1, a7
+; RV32IM-NEXT: add t1, t2, t1
+; RV32IM-NEXT: sltu t2, t1, t2
+; RV32IM-NEXT: add t1, t3, t1
+; RV32IM-NEXT: sltu t1, t1, t3
+; RV32IM-NEXT: mul t3, a2, t0
+; RV32IM-NEXT: add a5, a5, t2
+; RV32IM-NEXT: add a3, a3, t1
+; RV32IM-NEXT: srai t1, a5, 31
+; RV32IM-NEXT: srai t2, a3, 31
+; RV32IM-NEXT: mul a4, a4, a6
+; RV32IM-NEXT: add t1, t1, t2
+; RV32IM-NEXT: mulhsu a6, a1, a7
+; RV32IM-NEXT: add a3, a5, a3
+; RV32IM-NEXT: mulhsu a7, a2, t0
+; RV32IM-NEXT: sltu a5, a3, a5
+; RV32IM-NEXT: add a5, t1, a5
+; RV32IM-NEXT: add a5, t4, a5
+; RV32IM-NEXT: add t5, t6, t5
+; RV32IM-NEXT: sltu t0, t5, t6
+; RV32IM-NEXT: add t5, t3, t5
+; RV32IM-NEXT: sltu t1, t5, t3
+; RV32IM-NEXT: mul t2, a1, a2
+; RV32IM-NEXT: add a3, a4, a3
+; RV32IM-NEXT: mulh a1, a1, a2
+; RV32IM-NEXT: add a6, a6, t0
+; RV32IM-NEXT: add a7, a7, t1
+; RV32IM-NEXT: sltu a2, a3, a4
+; RV32IM-NEXT: add a4, a6, a7
+; RV32IM-NEXT: srai t0, a6, 31
+; RV32IM-NEXT: srai a7, a7, 31
+; RV32IM-NEXT: sltu a6, a4, a6
+; RV32IM-NEXT: add a7, t0, a7
+; RV32IM-NEXT: add a6, a7, a6
+; RV32IM-NEXT: add a4, t2, a4
+; RV32IM-NEXT: add a1, a1, a6
+; RV32IM-NEXT: sltu a6, a4, t2
+; RV32IM-NEXT: add a2, a5, a2
+; RV32IM-NEXT: add a1, a1, a6
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a2, 4(a0)
+; RV32IM-NEXT: sw a4, 8(a0)
+; RV32IM-NEXT: sw a1, 12(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v2i64:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: mulh a0, a0, a2
+; RV64IM-NEXT: mulh a1, a1, a3
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v2i64:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <2 x i64> @llvm.smulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @smulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; RV32I-LABEL: smulh_v3i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a4, 0(a2)
+; RV32I-NEXT: lw s0, 4(a2)
+; RV32I-NEXT: lw s1, 8(a2)
+; RV32I-NEXT: lw a2, 0(a1)
+; RV32I-NEXT: lw s2, 4(a1)
+; RV32I-NEXT: lw s3, 8(a1)
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: srai a1, a2, 31
+; RV32I-NEXT: srai a3, a4, 31
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: mv a2, a4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: srai a1, s2, 31
+; RV32I-NEXT: srai a3, s0, 31
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: srai a1, s3, 31
+; RV32I-NEXT: srai a3, s1, 31
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw s5, 0(s4)
+; RV32I-NEXT: sw s0, 4(s4)
+; RV32I-NEXT: sw a1, 8(s4)
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v3i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -64
+; RV64I-NEXT: sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lw a3, 0(a1)
+; RV64I-NEXT: lw s0, 8(a1)
+; RV64I-NEXT: lw s1, 16(a1)
+; RV64I-NEXT: lw a1, 0(a2)
+; RV64I-NEXT: lw s2, 8(a2)
+; RV64I-NEXT: lw s3, 16(a2)
+; RV64I-NEXT: mv s4, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: slli a0, a0, 32
+; RV64I-NEXT: or s0, s5, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: sd s0, 0(s4)
+; RV64I-NEXT: sw a0, 8(s4)
+; RV64I-NEXT: ld ra, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 64
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v3i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a2)
+; RV32IM-NEXT: lw a4, 4(a2)
+; RV32IM-NEXT: lw a2, 8(a2)
+; RV32IM-NEXT: lw a5, 0(a1)
+; RV32IM-NEXT: lw a6, 4(a1)
+; RV32IM-NEXT: lw a1, 8(a1)
+; RV32IM-NEXT: mulh a3, a5, a3
+; RV32IM-NEXT: mulh a4, a6, a4
+; RV32IM-NEXT: mulh a1, a1, a2
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a4, 4(a0)
+; RV32IM-NEXT: sw a1, 8(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v3i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lw a3, 8(a2)
+; RV64IM-NEXT: lw a4, 8(a1)
+; RV64IM-NEXT: lw a5, 0(a1)
+; RV64IM-NEXT: lw a6, 0(a2)
+; RV64IM-NEXT: lw a2, 16(a2)
+; RV64IM-NEXT: lw a1, 16(a1)
+; RV64IM-NEXT: mul a3, a4, a3
+; RV64IM-NEXT: mul a4, a5, a6
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 32
+; RV64IM-NEXT: srli a4, a4, 32
+; RV64IM-NEXT: slli a3, a3, 32
+; RV64IM-NEXT: or a3, a4, a3
+; RV64IM-NEXT: srli a1, a1, 32
+; RV64IM-NEXT: sd a3, 0(a0)
+; RV64IM-NEXT: sw a1, 8(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v3i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <3 x i32> @llvm.smulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @smulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; RV32I-LABEL: smulh_v2i48:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -64
+; RV32I-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s9, 0(a1)
+; RV32I-NEXT: lw s10, 4(a1)
+; RV32I-NEXT: lw s11, 8(a1)
+; RV32I-NEXT: lw a1, 12(a1)
+; RV32I-NEXT: sw a1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s4, 0(a2)
+; RV32I-NEXT: lh s3, 4(a2)
+; RV32I-NEXT: lw s2, 8(a2)
+; RV32I-NEXT: lh s1, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: slli s5, s9, 16
+; RV32I-NEXT: srai s6, s3, 31
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: mv a3, s6
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: srli a0, s9, 16
+; RV32I-NEXT: slli s10, s10, 16
+; RV32I-NEXT: or s9, s10, a0
+; RV32I-NEXT: srai s10, s10, 31
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s10
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: add a2, s7, s5
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: sltu a2, a2, s7
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add a2, s8, a2
+; RV32I-NEXT: add s4, a0, a2
+; RV32I-NEXT: srai a2, a2, 31
+; RV32I-NEXT: srai a1, a0, 31
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s5, a1, a0
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s10
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: mv a3, s6
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s4
+; RV32I-NEXT: add a1, a1, s5
+; RV32I-NEXT: sltu s10, s9, a0
+; RV32I-NEXT: add s10, a1, s10
+; RV32I-NEXT: slli s3, s11, 16
+; RV32I-NEXT: srai s4, s1, 31
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: mv a3, s4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: srli a0, s11, 16
+; RV32I-NEXT: lw a1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: slli a1, a1, 16
+; RV32I-NEXT: or s7, a1, a0
+; RV32I-NEXT: srai s8, a1, 31
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a2, s5, s3
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: sltu a2, a2, s5
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add a2, s6, a2
+; RV32I-NEXT: add s2, a0, a2
+; RV32I-NEXT: srai a2, a2, 31
+; RV32I-NEXT: srai a1, a0, 31
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s3, a1, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: mv a3, s4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add a1, a1, s3
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: srli a2, s2, 16
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw s9, 0(s0)
+; RV32I-NEXT: sh s10, 4(s0)
+; RV32I-NEXT: sh s2, 6(s0)
+; RV32I-NEXT: sh a2, 8(s0)
+; RV32I-NEXT: sh a0, 10(s0)
+; RV32I-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 64
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v2i48:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a1
+; RV64I-NEXT: slli s3, a3, 16
+; RV64I-NEXT: slli a3, a2, 16
+; RV64I-NEXT: srai s1, s3, 16
+; RV64I-NEXT: srai a2, a3, 16
+; RV64I-NEXT: slli a0, a0, 16
+; RV64I-NEXT: srai a3, a3, 63
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: slli a0, s0, 16
+; RV64I-NEXT: srai a3, s3, 63
+; RV64I-NEXT: srai a1, a0, 63
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: smulh_v2i48:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a5, 0(a1)
+; RV32IM-NEXT: lw a6, 4(a1)
+; RV32IM-NEXT: lw a7, 0(a2)
+; RV32IM-NEXT: lh a4, 4(a2)
+; RV32IM-NEXT: lw a3, 8(a2)
+; RV32IM-NEXT: lh a2, 12(a2)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: srli t1, a5, 16
+; RV32IM-NEXT: slli a6, a6, 16
+; RV32IM-NEXT: slli a5, a5, 16
+; RV32IM-NEXT: or a6, a6, t1
+; RV32IM-NEXT: mulhu t1, a5, a7
+; RV32IM-NEXT: mul t2, a6, a7
+; RV32IM-NEXT: mul t3, a4, a5
+; RV32IM-NEXT: slli t4, t0, 16
+; RV32IM-NEXT: srli t0, t0, 16
+; RV32IM-NEXT: mulhsu a5, a4, a5
+; RV32IM-NEXT: mulhsu a7, a6, a7
+; RV32IM-NEXT: slli a1, a1, 16
+; RV32IM-NEXT: mulhu t5, t4, a3
+; RV32IM-NEXT: or a1, a1, t0
+; RV32IM-NEXT: mul t0, a2, t4
+; RV32IM-NEXT: mulh t6, a6, a4
+; RV32IM-NEXT: add t1, t2, t1
+; RV32IM-NEXT: sltu t2, t1, t2
+; RV32IM-NEXT: add t1, t3, t1
+; RV32IM-NEXT: sltu t1, t1, t3
+; RV32IM-NEXT: mul t3, a1, a3
+; RV32IM-NEXT: add a7, a7, t2
+; RV32IM-NEXT: add a5, a5, t1
+; RV32IM-NEXT: srai t1, a7, 31
+; RV32IM-NEXT: srai t2, a5, 31
+; RV32IM-NEXT: add t1, t1, t2
+; RV32IM-NEXT: add a5, a7, a5
+; RV32IM-NEXT: mul a4, a6, a4
+; RV32IM-NEXT: sltu a6, a5, a7
+; RV32IM-NEXT: mulhsu a3, a1, a3
+; RV32IM-NEXT: mulhsu a7, a2, t4
+; RV32IM-NEXT: add a6, t1, a6
+; RV32IM-NEXT: add a6, t6, a6
+; RV32IM-NEXT: add t5, t3, t5
+; RV32IM-NEXT: sltu t1, t5, t3
+; RV32IM-NEXT: add t5, t0, t5
+; RV32IM-NEXT: sltu t0, t5, t0
+; RV32IM-NEXT: mul t2, a1, a2
+; RV32IM-NEXT: add a5, a4, a5
+; RV32IM-NEXT: mulh a1, a1, a2
+; RV32IM-NEXT: add a3, a3, t1
+; RV32IM-NEXT: add a7, a7, t0
+; RV32IM-NEXT: sltu a2, a5, a4
+; RV32IM-NEXT: add a4, a3, a7
+; RV32IM-NEXT: srai t0, a3, 31
+; RV32IM-NEXT: srai a7, a7, 31
+; RV32IM-NEXT: sltu a3, a4, a3
+; RV32IM-NEXT: add a7, t0, a7
+; RV32IM-NEXT: add a3, a7, a3
+; RV32IM-NEXT: add a4, t2, a4
+; RV32IM-NEXT: add a1, a1, a3
+; RV32IM-NEXT: sltu a3, a4, t2
+; RV32IM-NEXT: add a2, a6, a2
+; RV32IM-NEXT: add a1, a1, a3
+; RV32IM-NEXT: srli a3, a4, 16
+; RV32IM-NEXT: sw a5, 0(a0)
+; RV32IM-NEXT: sh a2, 4(a0)
+; RV32IM-NEXT: sh a4, 6(a0)
+; RV32IM-NEXT: sh a3, 8(a0)
+; RV32IM-NEXT: sh a1, 10(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: smulh_v2i48:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: slli a3, a3, 16
+; RV64IM-NEXT: slli a2, a2, 16
+; RV64IM-NEXT: srai a3, a3, 16
+; RV64IM-NEXT: srai a2, a2, 16
+; RV64IM-NEXT: slli a0, a0, 16
+; RV64IM-NEXT: slli a1, a1, 16
+; RV64IM-NEXT: mulh a0, a0, a2
+; RV64IM-NEXT: mulh a1, a1, a3
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: smulh_v2i48:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-V-NEXT: vsll.vi v9, v9, 16
+; CHECK-V-NEXT: vsra.vi v9, v9, 16
+; CHECK-V-NEXT: vsll.vi v8, v8, 16
+; CHECK-V-NEXT: vmulh.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <2 x i48> @llvm.smulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @smulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; RV32I-LABEL: smulh_v2i128:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -128
+; RV32I-NEXT: sw ra, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 120(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 116(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 112(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 104(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 100(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 96(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 16(a2)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 20(a2)
+; RV32I-NEXT: sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 24(a2)
+; RV32I-NEXT: sw a3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 28(a2)
+; RV32I-NEXT: sw a3, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 16(a1)
+; RV32I-NEXT: sw a3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 20(a1)
+; RV32I-NEXT: sw a3, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 24(a1)
+; RV32I-NEXT: sw a3, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 28(a1)
+; RV32I-NEXT: sw a3, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s7, 0(a2)
+; RV32I-NEXT: lw s5, 4(a2)
+; RV32I-NEXT: lw a3, 8(a2)
+; RV32I-NEXT: sw a3, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a2, 12(a2)
+; RV32I-NEXT: sw a2, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s3, 0(a1)
+; RV32I-NEXT: lw s1, 4(a1)
+; RV32I-NEXT: lw s0, 8(a1)
+; RV32I-NEXT: lw s11, 12(a1)
+; RV32I-NEXT: sw a0, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s4
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add s2, a1, a0
+; RV32I-NEXT: sw s0, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s10, a0, s4
+; RV32I-NEXT: sltu a0, s10, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s4, s2, a0
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: add s9, a0, s4
+; RV32I-NEXT: sw s11, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: srai a2, s11, 31
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: sw a2, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s4, s2
+; RV32I-NEXT: sltu a3, s9, s0
+; RV32I-NEXT: add a2, s8, a2
+; RV32I-NEXT: add s8, s9, a0
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sltu s9, s8, s9
+; RV32I-NEXT: add a1, a2, a1
+; RV32I-NEXT: add s9, a1, s9
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s2, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s2, a0
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s0
+; RV32I-NEXT: sltu a2, s0, s2
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s4, s6, s4
+; RV32I-NEXT: add s2, s10, a0
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, s4, s6
+; RV32I-NEXT: add s2, s2, a0
+; RV32I-NEXT: beq s2, s10, .LBB15_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu a0, s2, s10
+; RV32I-NEXT: .LBB15_2:
+; RV32I-NEXT: add s10, s8, a0
+; RV32I-NEXT: sltu s8, s10, s8
+; RV32I-NEXT: add s8, s9, s8
+; RV32I-NEXT: sw s8, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s4, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s8, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s5, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s0
+; RV32I-NEXT: sltu a0, s9, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s11, s8, a0
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: add s0, a0, s11
+; RV32I-NEXT: srai s5, s5, 31
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s0, s4
+; RV32I-NEXT: add a0, s0, a0
+; RV32I-NEXT: sltu a3, s11, s8
+; RV32I-NEXT: sltu s4, a0, s0
+; RV32I-NEXT: add a3, s7, a3
+; RV32I-NEXT: add a3, a3, a2
+; RV32I-NEXT: lw a2, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a2, s6, a2
+; RV32I-NEXT: add s2, s9, s2
+; RV32I-NEXT: sltu a2, a2, s6
+; RV32I-NEXT: add a3, a3, a1
+; RV32I-NEXT: add a1, s2, a2
+; RV32I-NEXT: add s4, a3, s4
+; RV32I-NEXT: beq a1, s9, .LBB15_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu a2, a1, s9
+; RV32I-NEXT: .LBB15_4:
+; RV32I-NEXT: add a2, a0, a2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s4, s4, a0
+; RV32I-NEXT: add s8, s10, a2
+; RV32I-NEXT: lw s11, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s7, s11, s4
+; RV32I-NEXT: sltu s2, s8, s10
+; RV32I-NEXT: add s7, s7, s2
+; RV32I-NEXT: lw s10, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: lw a0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: add s9, a0, s6
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s8, s0, s8
+; RV32I-NEXT: add a2, a0, s9
+; RV32I-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a4, s8, s0
+; RV32I-NEXT: add a3, a2, a4
+; RV32I-NEXT: add a3, a3, s7
+; RV32I-NEXT: beq a3, a2, .LBB15_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu a4, a3, a2
+; RV32I-NEXT: .LBB15_6:
+; RV32I-NEXT: sw a4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: beq s7, s11, .LBB15_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s2, s7, s11
+; RV32I-NEXT: .LBB15_8:
+; RV32I-NEXT: srai s7, s11, 31
+; RV32I-NEXT: srai a3, s4, 31
+; RV32I-NEXT: sltu a4, s9, s3
+; RV32I-NEXT: add s8, s7, a3
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s9, s1, a4
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s10, s8, s2
+; RV32I-NEXT: add s11, s9, a0
+; RV32I-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s2, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: add s0, a0, s11
+; RV32I-NEXT: lw a0, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: lw a2, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: lw a2, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, s4
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s8, s7
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: add a2, s8, a2
+; RV32I-NEXT: add s4, s0, s2
+; RV32I-NEXT: sltu a3, s10, s8
+; RV32I-NEXT: add s10, s4, s10
+; RV32I-NEXT: sw s10, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sltu s10, s4, s0
+; RV32I-NEXT: sltu a3, s11, s9
+; RV32I-NEXT: sltu a4, s0, s6
+; RV32I-NEXT: add a3, s1, a3
+; RV32I-NEXT: add a1, a1, s3
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a3, a3, a4
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add a0, a3, a0
+; RV32I-NEXT: sw a0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add a2, s10, a2
+; RV32I-NEXT: sw a2, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s3, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw s7, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s2, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s8, a0, s0
+; RV32I-NEXT: sltu a0, s8, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s5, s2, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: add s6, a0, s5
+; RV32I-NEXT: srai a2, s7, 31
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s11
+; RV32I-NEXT: sw a2, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s5, s2
+; RV32I-NEXT: sltu a3, s6, s0
+; RV32I-NEXT: add a2, s3, a2
+; RV32I-NEXT: add s3, s6, a0
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: sltu s9, s3, s6
+; RV32I-NEXT: add a1, a2, a1
+; RV32I-NEXT: add s9, a1, s9
+; RV32I-NEXT: lw s6, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw s5, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s2, a1, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s2, a0
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, a0, s0
+; RV32I-NEXT: sltu a2, s0, s2
+; RV32I-NEXT: sltu a0, s7, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s7, s1, s7
+; RV32I-NEXT: add s5, s8, a0
+; RV32I-NEXT: sw s7, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, s7, s1
+; RV32I-NEXT: add s5, s5, a0
+; RV32I-NEXT: beq s5, s8, .LBB15_10
+; RV32I-NEXT: # %bb.9:
+; RV32I-NEXT: sltu a0, s5, s8
+; RV32I-NEXT: .LBB15_10:
+; RV32I-NEXT: lw a2, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a1, a2, a1
+; RV32I-NEXT: sw a1, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s6, s3, a0
+; RV32I-NEXT: lw a0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: sw a0, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, s6, s3
+; RV32I-NEXT: sltu a1, a2, s4
+; RV32I-NEXT: sw a1, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add a0, s9, a0
+; RV32I-NEXT: sw a0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw s9, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s8, a1, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s0
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s10, s8, a0
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: add s11, a0, s10
+; RV32I-NEXT: srai s1, s1, 31
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: mv a3, s9
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s11, s0
+; RV32I-NEXT: add a0, s11, a0
+; RV32I-NEXT: sltu a3, s10, s8
+; RV32I-NEXT: sltu s11, a0, s11
+; RV32I-NEXT: add a3, s4, a3
+; RV32I-NEXT: add a3, a3, a2
+; RV32I-NEXT: lw s7, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s7, s3, s7
+; RV32I-NEXT: add s5, s2, s5
+; RV32I-NEXT: sltu a2, s7, s3
+; RV32I-NEXT: add a3, a3, a1
+; RV32I-NEXT: add a1, s5, a2
+; RV32I-NEXT: add s11, a3, s11
+; RV32I-NEXT: beq a1, s2, .LBB15_12
+; RV32I-NEXT: # %bb.11:
+; RV32I-NEXT: sltu a2, a1, s2
+; RV32I-NEXT: .LBB15_12:
+; RV32I-NEXT: add a2, a0, a2
+; RV32I-NEXT: lw a1, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a3, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sltu a1, a1, a3
+; RV32I-NEXT: sw a1, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s11, s11, a0
+; RV32I-NEXT: add s7, s6, a2
+; RV32I-NEXT: lw a0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a0, a0, s11
+; RV32I-NEXT: sltu s8, s7, s6
+; RV32I-NEXT: lw s10, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s10, s10, a1
+; RV32I-NEXT: add s6, a0, s8
+; RV32I-NEXT: lw s9, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s2, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: add s4, a0, s5
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, s0, s7
+; RV32I-NEXT: add a2, a0, s4
+; RV32I-NEXT: sw s7, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu s7, s7, s0
+; RV32I-NEXT: add s5, a2, s7
+; RV32I-NEXT: add s5, s5, s6
+; RV32I-NEXT: beq s5, a2, .LBB15_14
+; RV32I-NEXT: # %bb.13:
+; RV32I-NEXT: sltu s7, s5, a2
+; RV32I-NEXT: .LBB15_14:
+; RV32I-NEXT: sw s5, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s10, s10, a3
+; RV32I-NEXT: sw s10, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: beq s6, a3, .LBB15_16
+; RV32I-NEXT: # %bb.15:
+; RV32I-NEXT: sltu s8, s6, a3
+; RV32I-NEXT: .LBB15_16:
+; RV32I-NEXT: srai s6, a3, 31
+; RV32I-NEXT: srai a3, s11, 31
+; RV32I-NEXT: sltu a4, s4, s2
+; RV32I-NEXT: add s9, s6, a3
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s3, s3, a4
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s8, s9, s8
+; RV32I-NEXT: add s11, s3, a0
+; RV32I-NEXT: lw s10, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: sw a1, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s0, a0, s11
+; RV32I-NEXT: lw a0, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: lw a2, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, a2
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: lw a2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a3, s10
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s9, s6
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: sltu a3, s8, s9
+; RV32I-NEXT: add a4, s0, s2
+; RV32I-NEXT: add a2, s9, a2
+; RV32I-NEXT: add s8, a4, s8
+; RV32I-NEXT: add a2, a2, a3
+; RV32I-NEXT: add s7, s8, s7
+; RV32I-NEXT: sltu a3, a4, s0
+; RV32I-NEXT: sltu a5, s11, s3
+; RV32I-NEXT: sltu a6, s0, s5
+; RV32I-NEXT: lw a7, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a5, a7, a5
+; RV32I-NEXT: add a1, a1, s4
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a5, a5, a6
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add a0, a5, a0
+; RV32I-NEXT: add a2, a3, a2
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: sltu a1, s8, a4
+; RV32I-NEXT: sltu a2, s7, s8
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: lw a1, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a2, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a2, 16(a1)
+; RV32I-NEXT: lw a2, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a2, 20(a1)
+; RV32I-NEXT: sw s7, 24(a1)
+; RV32I-NEXT: sw a0, 28(a1)
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 0(a1)
+; RV32I-NEXT: lw a0, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 4(a1)
+; RV32I-NEXT: lw a0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 8(a1)
+; RV32I-NEXT: lw a0, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 12(a1)
+; RV32I-NEXT: lw ra, 124(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 116(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 112(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 108(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 100(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 96(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 128
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: smulh_v2i128:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -112
+; RV64I-NEXT: sd ra, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: ld s5, 0(a2)
+; RV64I-NEXT: ld s6, 8(a2)
+; RV64I-NEXT: ld s4, 16(a2)
+; RV64I-NEXT: ld s1, 24(a2)
+; RV64I-NEXT: ld s7, 0(a1)
+; RV64I-NEXT: ld s8, 8(a1)
+; RV64I-NEXT: ld a2, 16(a1)
+; RV64I-NEXT: sd a2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: ld s2, 24(a1)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: srai s9, s8, 63
+; RV64I-NEXT: mv a0, s8
+; RV64I-NEXT: mv a1, s9
+; RV64I-NEXT: mv a2, s5
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s10, a0
+; RV64I-NEXT: mv s11, a1
+; RV64I-NEXT: mv a0, s7
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s5
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s3, s10, a1
+; RV64I-NEXT: sltu a0, s3, s10
+; RV64I-NEXT: add s11, s11, a0
+; RV64I-NEXT: srai s5, s6, 63
+; RV64I-NEXT: mv a0, s7
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s6
+; RV64I-NEXT: mv a3, s5
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s3, a0, s3
+; RV64I-NEXT: sltu a0, s3, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s3, s11, a0
+; RV64I-NEXT: srai a1, s11, 63
+; RV64I-NEXT: srai a0, a0, 63
+; RV64I-NEXT: sltu a2, s3, s11
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s7, a0, a2
+; RV64I-NEXT: mv a0, s8
+; RV64I-NEXT: mv a1, s9
+; RV64I-NEXT: mv a2, s6
+; RV64I-NEXT: mv a3, s5
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s8, a0, s3
+; RV64I-NEXT: add a1, a1, s7
+; RV64I-NEXT: sltu s9, s8, a0
+; RV64I-NEXT: add s9, a1, s9
+; RV64I-NEXT: srai s5, s2, 63
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: mv a2, s4
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv s7, a1
+; RV64I-NEXT: ld s10, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s4
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s3, s6, a1
+; RV64I-NEXT: sltu a0, s3, s6
+; RV64I-NEXT: add s7, s7, a0
+; RV64I-NEXT: srai s4, s1, 63
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: mv a3, s4
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s3, a0, s3
+; RV64I-NEXT: sltu a0, s3, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s3, s7, a0
+; RV64I-NEXT: srai a1, s7, 63
+; RV64I-NEXT: srai a0, a0, 63
+; RV64I-NEXT: sltu a2, s3, s7
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s6, a0, a2
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: mv a3, s4
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s3, a0, s3
+; RV64I-NEXT: add a1, a1, s6
+; RV64I-NEXT: sltu a0, s3, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: sd s8, 0(s0)
+; RV64I-NEXT: sd s9, 8(s0)
+; RV64I-NEXT: sd s3, 16(s0)
+; RV64I-NEXT: sd a0, 24(s0)
+; RV64I-NEXT: ld ra, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 112
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: smulh_v2i128:
+; RV32M: # %bb.0:
+; RV32M-NEXT: addi sp, sp, -64
+; RV32M-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s9, 20(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s10, 16(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s11, 12(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw a0, 8(sp) # 4-byte Folded Spill
+; RV32M-NEXT: lw t0, 8(a1)
+; RV32M-NEXT: lw a7, 12(a1)
+; RV32M-NEXT: lw a5, 0(a2)
+; RV32M-NEXT: lw a6, 4(a2)
+; RV32M-NEXT: lw t2, 8(a2)
+; RV32M-NEXT: lw t1, 12(a2)
+; RV32M-NEXT: mulhu t4, t0, a5
+; RV32M-NEXT: mul t6, a7, a5
+; RV32M-NEXT: mul s0, t0, a6
+; RV32M-NEXT: lw a3, 0(a1)
+; RV32M-NEXT: lw a4, 4(a1)
+; RV32M-NEXT: mulhu s1, a7, a5
+; RV32M-NEXT: mulhu s2, t0, a6
+; RV32M-NEXT: mul s3, a7, a6
+; RV32M-NEXT: srai t3, a7, 31
+; RV32M-NEXT: mulhu s4, a7, a6
+; RV32M-NEXT: mulhu s5, a3, a5
+; RV32M-NEXT: mul s6, a5, t3
+; RV32M-NEXT: mulhu s7, a5, t3
+; RV32M-NEXT: add t4, t6, t4
+; RV32M-NEXT: mul s8, a6, t3
+; RV32M-NEXT: add t5, s0, t4
+; RV32M-NEXT: sltu t4, t4, t6
+; RV32M-NEXT: sltu t6, t5, s0
+; RV32M-NEXT: add t4, s1, t4
+; RV32M-NEXT: add t6, s2, t6
+; RV32M-NEXT: mul s0, a4, a5
+; RV32M-NEXT: add t6, t4, t6
+; RV32M-NEXT: mul s1, a3, a6
+; RV32M-NEXT: add s2, s3, t6
+; RV32M-NEXT: sltu t4, t6, t4
+; RV32M-NEXT: sltu t6, s2, s3
+; RV32M-NEXT: add t4, s4, t4
+; RV32M-NEXT: mulhu s3, a4, a5
+; RV32M-NEXT: add s7, s7, s6
+; RV32M-NEXT: mulhu s4, a3, a6
+; RV32M-NEXT: add t4, t4, t6
+; RV32M-NEXT: add s7, s7, s8
+; RV32M-NEXT: add s5, s0, s5
+; RV32M-NEXT: mul t6, a4, a6
+; RV32M-NEXT: sltu s0, s5, s0
+; RV32M-NEXT: mulhu s8, a4, a6
+; RV32M-NEXT: add s5, s1, s5
+; RV32M-NEXT: mul s9, t0, a5
+; RV32M-NEXT: add s0, s3, s0
+; RV32M-NEXT: sltu a5, s5, s1
+; RV32M-NEXT: add a5, s4, a5
+; RV32M-NEXT: add a6, s2, s6
+; RV32M-NEXT: sltu s3, a6, s2
+; RV32M-NEXT: add a5, s0, a5
+; RV32M-NEXT: sltu s0, a5, s0
+; RV32M-NEXT: add a5, t6, a5
+; RV32M-NEXT: sltu t6, a5, t6
+; RV32M-NEXT: add s0, s8, s0
+; RV32M-NEXT: add t6, s0, t6
+; RV32M-NEXT: add a5, s9, a5
+; RV32M-NEXT: add s0, t5, t6
+; RV32M-NEXT: sltu t6, a5, s9
+; RV32M-NEXT: add s7, t4, s7
+; RV32M-NEXT: add t4, s0, t6
+; RV32M-NEXT: add s3, s7, s3
+; RV32M-NEXT: beq t4, t5, .LBB15_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu t6, t4, t5
+; RV32M-NEXT: .LBB15_2:
+; RV32M-NEXT: mulhu t5, a3, t2
+; RV32M-NEXT: mul s0, a4, t2
+; RV32M-NEXT: mulhu s1, a4, t2
+; RV32M-NEXT: mul s4, a3, t1
+; RV32M-NEXT: mulhu s5, a3, t1
+; RV32M-NEXT: mul s6, a4, t1
+; RV32M-NEXT: add t5, s0, t5
+; RV32M-NEXT: srai s2, t1, 31
+; RV32M-NEXT: sltu s0, t5, s0
+; RV32M-NEXT: mulhu s7, a4, t1
+; RV32M-NEXT: add s0, s1, s0
+; RV32M-NEXT: mul s1, s2, a3
+; RV32M-NEXT: add t5, s4, t5
+; RV32M-NEXT: mul s8, s2, a4
+; RV32M-NEXT: sltu a4, t5, s4
+; RV32M-NEXT: add s5, s5, a4
+; RV32M-NEXT: mulhu s4, s2, a3
+; RV32M-NEXT: add a4, a6, t6
+; RV32M-NEXT: mul t6, a3, t2
+; RV32M-NEXT: sltu s9, a4, a6
+; RV32M-NEXT: add s5, s0, s5
+; RV32M-NEXT: sltu a3, s5, s0
+; RV32M-NEXT: add s5, s6, s5
+; RV32M-NEXT: add s7, s7, a3
+; RV32M-NEXT: add a3, s5, s1
+; RV32M-NEXT: sltu a6, s5, s6
+; RV32M-NEXT: sltu s0, a3, s5
+; RV32M-NEXT: add a6, s7, a6
+; RV32M-NEXT: add s4, s4, s8
+; RV32M-NEXT: add s1, s4, s1
+; RV32M-NEXT: add a5, t6, a5
+; RV32M-NEXT: add t4, t5, t4
+; RV32M-NEXT: sltu a5, a5, t6
+; RV32M-NEXT: add s1, a6, s1
+; RV32M-NEXT: add a6, t4, a5
+; RV32M-NEXT: add s3, s3, s9
+; RV32M-NEXT: add s8, s1, s0
+; RV32M-NEXT: beq a6, t5, .LBB15_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu a5, a6, t5
+; RV32M-NEXT: .LBB15_4:
+; RV32M-NEXT: mulhu s9, t0, t2
+; RV32M-NEXT: mul s5, a7, t2
+; RV32M-NEXT: mul s7, t0, t1
+; RV32M-NEXT: mul t4, t0, t2
+; RV32M-NEXT: add a5, a3, a5
+; RV32M-NEXT: sltu a3, a5, a3
+; RV32M-NEXT: add s8, s8, a3
+; RV32M-NEXT: add a3, a4, a5
+; RV32M-NEXT: add a6, s3, s8
+; RV32M-NEXT: sltu s4, a3, a4
+; RV32M-NEXT: add a6, a6, s4
+; RV32M-NEXT: add s9, s5, s9
+; RV32M-NEXT: add s10, s7, s9
+; RV32M-NEXT: add a3, t4, a3
+; RV32M-NEXT: add a4, s10, a6
+; RV32M-NEXT: sw a3, 4(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sltu a0, a3, t4
+; RV32M-NEXT: add a4, a4, a0
+; RV32M-NEXT: beq a4, s10, .LBB15_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu a0, a4, s10
+; RV32M-NEXT: .LBB15_6:
+; RV32M-NEXT: sw a4, 0(sp) # 4-byte Folded Spill
+; RV32M-NEXT: lw s0, 16(a2)
+; RV32M-NEXT: lw s1, 20(a2)
+; RV32M-NEXT: lw t4, 16(a1)
+; RV32M-NEXT: lw t5, 20(a1)
+; RV32M-NEXT: lw a5, 24(a1)
+; RV32M-NEXT: lw a1, 28(a1)
+; RV32M-NEXT: beq a6, s3, .LBB15_8
+; RV32M-NEXT: # %bb.7:
+; RV32M-NEXT: sltu s4, a6, s3
+; RV32M-NEXT: .LBB15_8:
+; RV32M-NEXT: mulhu s11, a7, t2
+; RV32M-NEXT: mulhu ra, t0, t1
+; RV32M-NEXT: mul a3, a7, t1
+; RV32M-NEXT: mul a4, t2, t3
+; RV32M-NEXT: mul s6, s2, t0
+; RV32M-NEXT: lw a6, 24(a2)
+; RV32M-NEXT: lw a2, 28(a2)
+; RV32M-NEXT: srai s3, s3, 31
+; RV32M-NEXT: srai s8, s8, 31
+; RV32M-NEXT: sltu s5, s9, s5
+; RV32M-NEXT: sltu s7, s10, s7
+; RV32M-NEXT: mulhu s9, t2, t3
+; RV32M-NEXT: add s5, s11, s5
+; RV32M-NEXT: add s7, ra, s7
+; RV32M-NEXT: add s8, s3, s8
+; RV32M-NEXT: add s7, s5, s7
+; RV32M-NEXT: add s10, s6, a4
+; RV32M-NEXT: add s11, a3, s7
+; RV32M-NEXT: add s4, s8, s4
+; RV32M-NEXT: add t2, s11, s10
+; RV32M-NEXT: sltu s3, s8, s3
+; RV32M-NEXT: mul ra, t1, t3
+; RV32M-NEXT: add t3, s8, s3
+; RV32M-NEXT: sltu s3, s4, s8
+; RV32M-NEXT: mul s8, s2, a7
+; RV32M-NEXT: mulhu t6, s2, t0
+; RV32M-NEXT: add t0, t2, s4
+; RV32M-NEXT: add t3, t3, s3
+; RV32M-NEXT: sltu s2, t2, s11
+; RV32M-NEXT: mulhu a7, a7, t1
+; RV32M-NEXT: sltu a3, s11, a3
+; RV32M-NEXT: add a4, s9, a4
+; RV32M-NEXT: sltu t1, s7, s5
+; RV32M-NEXT: add a4, a4, ra
+; RV32M-NEXT: mulhu s3, a5, s0
+; RV32M-NEXT: mul s4, a1, s0
+; RV32M-NEXT: add t6, t6, s8
+; RV32M-NEXT: mul s5, a5, s1
+; RV32M-NEXT: add t6, t6, s6
+; RV32M-NEXT: mulhu s7, a1, s0
+; RV32M-NEXT: add a7, a7, t1
+; RV32M-NEXT: add a4, t6, a4
+; RV32M-NEXT: sltu t1, s10, s6
+; RV32M-NEXT: mulhu t6, a5, s1
+; RV32M-NEXT: add a3, a7, a3
+; RV32M-NEXT: add a4, a4, t1
+; RV32M-NEXT: add s3, s4, s3
+; RV32M-NEXT: mul s6, a1, s1
+; RV32M-NEXT: sltu a7, s3, s4
+; RV32M-NEXT: add s4, s5, s3
+; RV32M-NEXT: add s7, s7, a7
+; RV32M-NEXT: sltu t1, s4, s5
+; RV32M-NEXT: srai a7, a1, 31
+; RV32M-NEXT: mulhu s3, a1, s1
+; RV32M-NEXT: add t6, t6, t1
+; RV32M-NEXT: mul s5, s0, a7
+; RV32M-NEXT: add t1, a3, a4
+; RV32M-NEXT: add t6, s7, t6
+; RV32M-NEXT: add a3, s6, t6
+; RV32M-NEXT: mulhu a4, s0, a7
+; RV32M-NEXT: mulhu s8, t4, s0
+; RV32M-NEXT: mul s9, t5, s0
+; RV32M-NEXT: sltu s6, a3, s6
+; RV32M-NEXT: sltu t6, t6, s7
+; RV32M-NEXT: add t6, s3, t6
+; RV32M-NEXT: mul s7, t4, s1
+; RV32M-NEXT: add s3, a3, s5
+; RV32M-NEXT: add t6, t6, s6
+; RV32M-NEXT: mulhu s6, t5, s0
+; RV32M-NEXT: mulhu s10, t4, s1
+; RV32M-NEXT: add a4, a4, s5
+; RV32M-NEXT: mul s5, s1, a7
+; RV32M-NEXT: add s8, s9, s8
+; RV32M-NEXT: mul s11, t5, s1
+; RV32M-NEXT: sltu s9, s8, s9
+; RV32M-NEXT: mulhu s1, t5, s1
+; RV32M-NEXT: add s8, s7, s8
+; RV32M-NEXT: mul s0, a5, s0
+; RV32M-NEXT: sltu a3, s3, a3
+; RV32M-NEXT: sltu s7, s8, s7
+; RV32M-NEXT: add s6, s6, s9
+; RV32M-NEXT: add s7, s10, s7
+; RV32M-NEXT: add a4, a4, s5
+; RV32M-NEXT: add s7, s6, s7
+; RV32M-NEXT: add s5, s11, s7
+; RV32M-NEXT: sltu s6, s7, s6
+; RV32M-NEXT: sltu s7, s5, s11
+; RV32M-NEXT: add s1, s1, s6
+; RV32M-NEXT: add s1, s1, s7
+; RV32M-NEXT: add s5, s0, s5
+; RV32M-NEXT: add s1, s4, s1
+; RV32M-NEXT: sltu s7, s5, s0
+; RV32M-NEXT: add a4, t6, a4
+; RV32M-NEXT: add s6, s1, s7
+; RV32M-NEXT: add s0, s2, t3
+; RV32M-NEXT: add t3, a4, a3
+; RV32M-NEXT: beq s6, s4, .LBB15_10
+; RV32M-NEXT: # %bb.9:
+; RV32M-NEXT: sltu s7, s6, s4
+; RV32M-NEXT: .LBB15_10:
+; RV32M-NEXT: mulhu a3, t4, a6
+; RV32M-NEXT: mul a4, t5, a6
+; RV32M-NEXT: mulhu s2, t5, a6
+; RV32M-NEXT: mul s8, t4, a2
+; RV32M-NEXT: add t6, t0, a0
+; RV32M-NEXT: add s0, t1, s0
+; RV32M-NEXT: sltu s1, t0, t2
+; RV32M-NEXT: mulhu a0, t4, a2
+; RV32M-NEXT: mul s9, t5, a2
+; RV32M-NEXT: add a3, a4, a3
+; RV32M-NEXT: srai t1, a2, 31
+; RV32M-NEXT: sltu a4, a3, a4
+; RV32M-NEXT: mulhu s10, t5, a2
+; RV32M-NEXT: add a4, s2, a4
+; RV32M-NEXT: mul s2, t1, t4
+; RV32M-NEXT: add s4, s8, a3
+; RV32M-NEXT: mul a3, t1, t5
+; RV32M-NEXT: sltu t2, s4, s8
+; RV32M-NEXT: add a0, a0, t2
+; RV32M-NEXT: mulhu t5, t1, t4
+; RV32M-NEXT: add t2, s3, s7
+; RV32M-NEXT: mul s7, t4, a6
+; RV32M-NEXT: sltu s8, t2, s3
+; RV32M-NEXT: add a0, a4, a0
+; RV32M-NEXT: sltu a4, a0, a4
+; RV32M-NEXT: add a0, s9, a0
+; RV32M-NEXT: add a4, s10, a4
+; RV32M-NEXT: add t4, a0, s2
+; RV32M-NEXT: sltu s3, a0, s9
+; RV32M-NEXT: sltu a0, t4, a0
+; RV32M-NEXT: add a4, a4, s3
+; RV32M-NEXT: add a3, t5, a3
+; RV32M-NEXT: add a3, a3, s2
+; RV32M-NEXT: add s5, s7, s5
+; RV32M-NEXT: add s3, s4, s6
+; RV32M-NEXT: sltu t5, s5, s7
+; RV32M-NEXT: add a3, a4, a3
+; RV32M-NEXT: add s3, s3, t5
+; RV32M-NEXT: add s2, t3, s8
+; RV32M-NEXT: add s5, a3, a0
+; RV32M-NEXT: beq s3, s4, .LBB15_12
+; RV32M-NEXT: # %bb.11:
+; RV32M-NEXT: sltu t5, s3, s4
+; RV32M-NEXT: .LBB15_12:
+; RV32M-NEXT: mulhu s6, a5, a6
+; RV32M-NEXT: mul s3, a1, a6
+; RV32M-NEXT: mul s4, a5, a2
+; RV32M-NEXT: mul a0, a5, a6
+; RV32M-NEXT: add t5, t4, t5
+; RV32M-NEXT: sltu a3, t5, t4
+; RV32M-NEXT: add s5, s5, a3
+; RV32M-NEXT: add a3, t2, t5
+; RV32M-NEXT: add s8, s2, s5
+; RV32M-NEXT: sltu t5, a3, t2
+; RV32M-NEXT: add s8, s8, t5
+; RV32M-NEXT: add s6, s3, s6
+; RV32M-NEXT: add s7, s4, s6
+; RV32M-NEXT: add t2, a0, a3
+; RV32M-NEXT: add t3, s7, s8
+; RV32M-NEXT: sltu t4, t2, a0
+; RV32M-NEXT: add t3, t3, t4
+; RV32M-NEXT: add s0, s0, s1
+; RV32M-NEXT: sltu t0, t6, t0
+; RV32M-NEXT: beq t3, s7, .LBB15_14
+; RV32M-NEXT: # %bb.13:
+; RV32M-NEXT: sltu t4, t3, s7
+; RV32M-NEXT: .LBB15_14:
+; RV32M-NEXT: add t0, s0, t0
+; RV32M-NEXT: beq s8, s2, .LBB15_16
+; RV32M-NEXT: # %bb.15:
+; RV32M-NEXT: sltu t5, s8, s2
+; RV32M-NEXT: .LBB15_16:
+; RV32M-NEXT: mulhu a0, a1, a6
+; RV32M-NEXT: mulhu a3, a5, a2
+; RV32M-NEXT: srai a4, s2, 31
+; RV32M-NEXT: mul s0, a1, a2
+; RV32M-NEXT: mul s1, a6, a7
+; RV32M-NEXT: mul s2, t1, a5
+; RV32M-NEXT: srai s5, s5, 31
+; RV32M-NEXT: sltu s3, s6, s3
+; RV32M-NEXT: sltu s4, s7, s4
+; RV32M-NEXT: mulhu s6, a1, a2
+; RV32M-NEXT: mulhu a6, a6, a7
+; RV32M-NEXT: mul a1, t1, a1
+; RV32M-NEXT: add a0, a0, s3
+; RV32M-NEXT: add a3, a3, s4
+; RV32M-NEXT: add s5, a4, s5
+; RV32M-NEXT: add a3, a0, a3
+; RV32M-NEXT: add s3, s2, s1
+; RV32M-NEXT: add s4, s0, a3
+; RV32M-NEXT: add t5, s5, t5
+; RV32M-NEXT: add s7, s4, s3
+; RV32M-NEXT: mulhu a5, t1, a5
+; RV32M-NEXT: add t1, s7, t5
+; RV32M-NEXT: sltu a4, s5, a4
+; RV32M-NEXT: mul a2, a2, a7
+; RV32M-NEXT: add a4, s5, a4
+; RV32M-NEXT: sltu a7, t5, s5
+; RV32M-NEXT: add t4, t1, t4
+; RV32M-NEXT: add a4, a4, a7
+; RV32M-NEXT: sltu a7, s7, s4
+; RV32M-NEXT: sltu a0, a3, a0
+; RV32M-NEXT: sltu a3, s4, s0
+; RV32M-NEXT: add a0, s6, a0
+; RV32M-NEXT: add a6, a6, s1
+; RV32M-NEXT: add a1, a5, a1
+; RV32M-NEXT: add a2, a6, a2
+; RV32M-NEXT: add a1, a1, s2
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: sltu a2, s3, s2
+; RV32M-NEXT: add a0, a0, a3
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: add a4, a7, a4
+; RV32M-NEXT: add a0, a0, a4
+; RV32M-NEXT: sltu a1, t1, s7
+; RV32M-NEXT: sltu a2, t4, t1
+; RV32M-NEXT: add a0, a0, a1
+; RV32M-NEXT: add a0, a0, a2
+; RV32M-NEXT: lw a1, 8(sp) # 4-byte Folded Reload
+; RV32M-NEXT: sw t2, 16(a1)
+; RV32M-NEXT: sw t3, 20(a1)
+; RV32M-NEXT: sw t4, 24(a1)
+; RV32M-NEXT: sw a0, 28(a1)
+; RV32M-NEXT: lw a0, 4(sp) # 4-byte Folded Reload
+; RV32M-NEXT: sw a0, 0(a1)
+; RV32M-NEXT: lw a0, 0(sp) # 4-byte Folded Reload
+; RV32M-NEXT: sw a0, 4(a1)
+; RV32M-NEXT: sw t6, 8(a1)
+; RV32M-NEXT: sw t0, 12(a1)
+; RV32M-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s8, 24(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s11, 12(sp) # 4-byte Folded Reload
+; RV32M-NEXT: addi sp, sp, 64
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: smulh_v2i128:
+; RV64M: # %bb.0:
+; RV64M-NEXT: ld a3, 0(a1)
+; RV64M-NEXT: ld a4, 8(a1)
+; RV64M-NEXT: ld a5, 0(a2)
+; RV64M-NEXT: ld a6, 8(a2)
+; RV64M-NEXT: ld a7, 16(a2)
+; RV64M-NEXT: ld a2, 24(a2)
+; RV64M-NEXT: ld t0, 16(a1)
+; RV64M-NEXT: ld a1, 24(a1)
+; RV64M-NEXT: mulhu t1, a3, a5
+; RV64M-NEXT: mul t2, a4, a5
+; RV64M-NEXT: mul t3, a6, a3
+; RV64M-NEXT: mulhsu a5, a4, a5
+; RV64M-NEXT: mulhsu a3, a6, a3
+; RV64M-NEXT: mulh t4, a4, a6
+; RV64M-NEXT: mulhu t5, t0, a7
+; RV64M-NEXT: mul t6, a1, a7
+; RV64M-NEXT: add t1, t2, t1
+; RV64M-NEXT: sltu t2, t1, t2
+; RV64M-NEXT: add t1, t3, t1
+; RV64M-NEXT: sltu t1, t1, t3
+; RV64M-NEXT: mul t3, a2, t0
+; RV64M-NEXT: add a5, a5, t2
+; RV64M-NEXT: add a3, a3, t1
+; RV64M-NEXT: srai t1, a5, 63
+; RV64M-NEXT: srai t2, a3, 63
+; RV64M-NEXT: mul a4, a4, a6
+; RV64M-NEXT: add t1, t1, t2
+; RV64M-NEXT: mulhsu a6, a1, a7
+; RV64M-NEXT: add a3, a5, a3
+; RV64M-NEXT: mulhsu a7, a2, t0
+; RV64M-NEXT: sltu a5, a3, a5
+; RV64M-NEXT: add a5, t1, a5
+; RV64M-NEXT: add a5, t4, a5
+; RV64M-NEXT: add t5, t6, t5
+; RV64M-NEXT: sltu t0, t5, t6
+; RV64M-NEXT: add t5, t3, t5
+; RV64M-NEXT: sltu t1, t5, t3
+; RV64M-NEXT: mul t2, a1, a2
+; RV64M-NEXT: add a3, a4, a3
+; RV64M-NEXT: mulh a1, a1, a2
+; RV64M-NEXT: add a6, a6, t0
+; RV64M-NEXT: add a7, a7, t1
+; RV64M-NEXT: sltu a2, a3, a4
+; RV64M-NEXT: add a4, a6, a7
+; RV64M-NEXT: srai t0, a6, 63
+; RV64M-NEXT: srai a7, a7, 63
+; RV64M-NEXT: sltu a6, a4, a6
+; RV64M-NEXT: add a7, t0, a7
+; RV64M-NEXT: add a6, a7, a6
+; RV64M-NEXT: add a4, t2, a4
+; RV64M-NEXT: add a1, a1, a6
+; RV64M-NEXT: sltu a6, a4, t2
+; RV64M-NEXT: add a2, a5, a2
+; RV64M-NEXT: add a1, a1, a6
+; RV64M-NEXT: sd a3, 0(a0)
+; RV64M-NEXT: sd a2, 8(a0)
+; RV64M-NEXT: sd a4, 16(a0)
+; RV64M-NEXT: sd a1, 24(a0)
+; RV64M-NEXT: ret
+ %r = call <2 x i128> @llvm.smulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; CHECK-I: {{.*}}
+; CHECK-M: {{.*}}
+; RV32V: {{.*}}
+; RV64V: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/umulh.ll b/llvm/test/CodeGen/RISCV/umulh.ll
new file mode 100644
index 0000000000000..6c1ac2cadc394
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/umulh.ll
@@ -0,0 +1,4319 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV32I
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s --check-prefixes=CHECK,CHECK-I,RV64I
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV32M,RV32IM
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV64M,RV64IM
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV32M,CHECK-V,RV32V
+; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK,CHECK-M,RV64M,CHECK-V,RV64V
+
+define i16 @umulh_i16(i16 %x, i16 %y) nounwind {
+; RV32I-LABEL: umulh_i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lui a2, 16
+; RV32I-NEXT: addi a2, a2, -1
+; RV32I-NEXT: and a0, a0, a2
+; RV32I-NEXT: and a1, a1, a2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 16
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lui a2, 16
+; RV64I-NEXT: addi a2, a2, -1
+; RV64I-NEXT: and a0, a0, a2
+; RV64I-NEXT: and a1, a1, a2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 16
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; CHECK-M-LABEL: umulh_i16:
+; CHECK-M: # %bb.0:
+; CHECK-M-NEXT: lui a2, 16
+; CHECK-M-NEXT: addi a2, a2, -1
+; CHECK-M-NEXT: and a1, a1, a2
+; CHECK-M-NEXT: and a0, a0, a2
+; CHECK-M-NEXT: mul a0, a0, a1
+; CHECK-M-NEXT: srli a0, a0, 16
+; CHECK-M-NEXT: ret
+ %r = call i16 @llvm.umulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @umulh_i32(i32 %x, i32 %y) nounwind {
+; RV32I-LABEL: umulh_i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a2, a1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv a0, a1
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: slli a0, a0, 32
+; RV64I-NEXT: slli a1, a1, 32
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: srli a1, a1, 32
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: umulh_i32:
+; RV32M: # %bb.0:
+; RV32M-NEXT: mulhu a0, a0, a1
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: umulh_i32:
+; RV64M: # %bb.0:
+; RV64M-NEXT: slli a1, a1, 32
+; RV64M-NEXT: slli a0, a0, 32
+; RV64M-NEXT: mulhu a0, a0, a1
+; RV64M-NEXT: srli a0, a0, 32
+; RV64M-NEXT: ret
+ %r = call i32 @llvm.umulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @umulh_i64(i64 %x, i64 %y) nounwind {
+; RV32I-LABEL: umulh_i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s4
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add s1, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s4
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s1, a0
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: sltu a3, s3, s1
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a1, a1, a3
+; RV32I-NEXT: add a1, a1, a0
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a2, a1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: umulh_i64:
+; RV32M: # %bb.0:
+; RV32M-NEXT: mulhu a4, a0, a2
+; RV32M-NEXT: mul a5, a1, a2
+; RV32M-NEXT: mul a6, a0, a3
+; RV32M-NEXT: mulhu a2, a1, a2
+; RV32M-NEXT: mulhu a0, a0, a3
+; RV32M-NEXT: mul a7, a1, a3
+; RV32M-NEXT: add a4, a5, a4
+; RV32M-NEXT: mulhu a1, a1, a3
+; RV32M-NEXT: add a3, a6, a4
+; RV32M-NEXT: sltu a4, a4, a5
+; RV32M-NEXT: sltu a3, a3, a6
+; RV32M-NEXT: add a2, a2, a4
+; RV32M-NEXT: add a0, a0, a3
+; RV32M-NEXT: add a3, a2, a0
+; RV32M-NEXT: add a0, a7, a3
+; RV32M-NEXT: sltu a2, a3, a2
+; RV32M-NEXT: sltu a3, a0, a7
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: umulh_i64:
+; RV64M: # %bb.0:
+; RV64M-NEXT: mulhu a0, a0, a1
+; RV64M-NEXT: ret
+ %r = call i64 @llvm.umulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @umulh_i96(i96 %x, i96 %y) nounwind {
+; RV32I-LABEL: umulh_i96:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s6, 0(a2)
+; RV32I-NEXT: lw s5, 4(a2)
+; RV32I-NEXT: lw a2, 8(a2)
+; RV32I-NEXT: sw a2, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s4, 0(a1)
+; RV32I-NEXT: lw s3, 4(a1)
+; RV32I-NEXT: lw s2, 8(a1)
+; RV32I-NEXT: sw a0, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, s7, a1
+; RV32I-NEXT: sltu a0, s0, s7
+; RV32I-NEXT: add s8, s8, a0
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s8, a0
+; RV32I-NEXT: sltu s1, s0, s8
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: add a1, a1, s1
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s1, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s8, a0
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: sw s2, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, s8, s0
+; RV32I-NEXT: add a2, a0, s9
+; RV32I-NEXT: add s1, a2, s1
+; RV32I-NEXT: sltu s10, s2, s8
+; RV32I-NEXT: add s3, s1, s10
+; RV32I-NEXT: beq s3, a2, .LBB3_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu s10, s3, a2
+; RV32I-NEXT: .LBB3_2:
+; RV32I-NEXT: sltu s0, s9, s7
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s0, s6, s0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s11, s0, a0
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: sw a1, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s6, a0, s11
+; RV32I-NEXT: add s10, s6, s10
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s8, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a0
+; RV32I-NEXT: add s8, s5, a1
+; RV32I-NEXT: li a0, 0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, s9, s2
+; RV32I-NEXT: add a2, a0, s8
+; RV32I-NEXT: add a3, a2, s3
+; RV32I-NEXT: sltu s2, s2, s9
+; RV32I-NEXT: add a3, a3, s2
+; RV32I-NEXT: beq a3, a2, .LBB3_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu s2, a3, a2
+; RV32I-NEXT: .LBB3_4:
+; RV32I-NEXT: sltu a3, s8, s5
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s7, s7, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s7, a0
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s6, s1
+; RV32I-NEXT: add a3, a0, s3
+; RV32I-NEXT: sltu a4, s11, s0
+; RV32I-NEXT: add s2, a3, s2
+; RV32I-NEXT: lw a5, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a4, a5, a4
+; RV32I-NEXT: add s9, s10, s2
+; RV32I-NEXT: sltu a5, s10, s6
+; RV32I-NEXT: add a2, a2, a5
+; RV32I-NEXT: sltu a5, s3, s7
+; RV32I-NEXT: add a1, a1, a5
+; RV32I-NEXT: sltu a0, a3, a0
+; RV32I-NEXT: sltu a3, s2, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, a4, a2
+; RV32I-NEXT: add a0, a0, a3
+; RV32I-NEXT: sltu s0, s9, s10
+; RV32I-NEXT: add a0, s1, a0
+; RV32I-NEXT: add s2, a0, s0
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s3, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: add s8, a0, s6
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, s5, s9
+; RV32I-NEXT: add a2, a0, s8
+; RV32I-NEXT: sltu s7, s6, s5
+; RV32I-NEXT: add s5, a2, s7
+; RV32I-NEXT: add s5, s5, s2
+; RV32I-NEXT: beq s5, a2, .LBB3_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu s7, s5, a2
+; RV32I-NEXT: .LBB3_6:
+; RV32I-NEXT: beq s2, s1, .LBB3_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s0, s2, s1
+; RV32I-NEXT: .LBB3_8:
+; RV32I-NEXT: sltu a3, s8, s4
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a3, s3, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, a3, a0
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a2, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: lw a1, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw s6, 0(a1)
+; RV32I-NEXT: add a0, a0, s0
+; RV32I-NEXT: add a0, a0, s7
+; RV32I-NEXT: add a0, a0, s1
+; RV32I-NEXT: sw s5, 4(a1)
+; RV32I-NEXT: sw a0, 8(a1)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_i96:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -64
+; RV64I-NEXT: sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a2
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: mv s2, a0
+; RV64I-NEXT: slli a3, a3, 32
+; RV64I-NEXT: srli s3, a3, 32
+; RV64I-NEXT: slli s4, a0, 32
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a0, s4
+; RV64I-NEXT: mv a2, s3
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s5, a0
+; RV64I-NEXT: mv s6, a1
+; RV64I-NEXT: mv a0, s4
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s4, a1
+; RV64I-NEXT: srli a0, s2, 32
+; RV64I-NEXT: slli s1, s1, 32
+; RV64I-NEXT: or s1, s1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s4, a0, s4
+; RV64I-NEXT: add a2, s5, s4
+; RV64I-NEXT: sltu a2, a2, s5
+; RV64I-NEXT: sltu a0, s4, a0
+; RV64I-NEXT: add a2, s6, a2
+; RV64I-NEXT: add s0, a1, a0
+; RV64I-NEXT: add s2, s0, a2
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s3
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add a2, a0, s2
+; RV64I-NEXT: sltu a3, s2, s0
+; RV64I-NEXT: sltu a0, a2, a0
+; RV64I-NEXT: add a1, a1, a3
+; RV64I-NEXT: add a1, a1, a0
+; RV64I-NEXT: mv a0, a2
+; RV64I-NEXT: ld ra, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 64
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: umulh_i96:
+; RV32M: # %bb.0:
+; RV32M-NEXT: lw a7, 0(a2)
+; RV32M-NEXT: lw a4, 4(a2)
+; RV32M-NEXT: lw a5, 0(a1)
+; RV32M-NEXT: lw a3, 4(a1)
+; RV32M-NEXT: lw a1, 8(a1)
+; RV32M-NEXT: lw a2, 8(a2)
+; RV32M-NEXT: mulhu a6, a5, a7
+; RV32M-NEXT: mul t0, a5, a4
+; RV32M-NEXT: mulhu t3, a3, a7
+; RV32M-NEXT: mul t1, a1, a7
+; RV32M-NEXT: mulhu t4, a5, a4
+; RV32M-NEXT: mul t2, a3, a4
+; RV32M-NEXT: mul t5, a3, a7
+; RV32M-NEXT: add a6, t0, a6
+; RV32M-NEXT: sltu t0, a6, t0
+; RV32M-NEXT: add t3, t1, t3
+; RV32M-NEXT: add t0, t4, t0
+; RV32M-NEXT: add t4, t2, t3
+; RV32M-NEXT: add a6, t5, a6
+; RV32M-NEXT: add t0, t4, t0
+; RV32M-NEXT: sltu a6, a6, t5
+; RV32M-NEXT: add t0, t0, a6
+; RV32M-NEXT: beq t0, t4, .LBB3_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu a6, t0, t4
+; RV32M-NEXT: .LBB3_2:
+; RV32M-NEXT: mulhu a7, a1, a7
+; RV32M-NEXT: mulhu t5, a3, a4
+; RV32M-NEXT: mul t6, a5, a2
+; RV32M-NEXT: sltu t1, t3, t1
+; RV32M-NEXT: mul t3, a1, a4
+; RV32M-NEXT: sltu t2, t4, t2
+; RV32M-NEXT: add a7, a7, t1
+; RV32M-NEXT: seqz t1, t0
+; RV32M-NEXT: mulhu a5, a5, a2
+; RV32M-NEXT: mulhu a4, a1, a4
+; RV32M-NEXT: addi t1, t1, -1
+; RV32M-NEXT: add t2, t5, t2
+; RV32M-NEXT: add t0, t6, t0
+; RV32M-NEXT: add t2, a7, t2
+; RV32M-NEXT: sltu t0, t0, t6
+; RV32M-NEXT: add t4, t3, t2
+; RV32M-NEXT: and t0, t1, t0
+; RV32M-NEXT: sltu a7, t2, a7
+; RV32M-NEXT: add a6, t4, a6
+; RV32M-NEXT: add t0, a5, t0
+; RV32M-NEXT: mul t2, a3, a2
+; RV32M-NEXT: mulhu t5, a3, a2
+; RV32M-NEXT: mul a3, a1, a2
+; RV32M-NEXT: add t6, a6, t0
+; RV32M-NEXT: add a7, a4, a7
+; RV32M-NEXT: sltu a4, t6, a6
+; RV32M-NEXT: sltu t1, t4, t3
+; RV32M-NEXT: add a7, a7, t1
+; RV32M-NEXT: sltu t1, a6, t4
+; RV32M-NEXT: add t1, a7, t1
+; RV32M-NEXT: sltu a5, t0, a5
+; RV32M-NEXT: add a6, t1, a5
+; RV32M-NEXT: add a5, t2, t6
+; RV32M-NEXT: add a7, a3, t5
+; RV32M-NEXT: sltu t0, a5, t2
+; RV32M-NEXT: add t2, a6, a4
+; RV32M-NEXT: add a6, a7, t0
+; RV32M-NEXT: add a6, a6, t2
+; RV32M-NEXT: beq a6, a7, .LBB3_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu t0, a6, a7
+; RV32M-NEXT: .LBB3_4:
+; RV32M-NEXT: beq t2, t1, .LBB3_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu a4, t2, t1
+; RV32M-NEXT: .LBB3_6:
+; RV32M-NEXT: mulhu a1, a1, a2
+; RV32M-NEXT: sltu a2, a7, a3
+; RV32M-NEXT: add a2, a2, a4
+; RV32M-NEXT: add a2, a2, t0
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: sw a5, 0(a0)
+; RV32M-NEXT: sw a6, 4(a0)
+; RV32M-NEXT: sw a1, 8(a0)
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: umulh_i96:
+; RV64M: # %bb.0:
+; RV64M-NEXT: slli a3, a3, 32
+; RV64M-NEXT: srli a3, a3, 32
+; RV64M-NEXT: srli a4, a0, 32
+; RV64M-NEXT: slli a1, a1, 32
+; RV64M-NEXT: slli a0, a0, 32
+; RV64M-NEXT: or a1, a1, a4
+; RV64M-NEXT: mulhu a4, a0, a2
+; RV64M-NEXT: mul a5, a1, a2
+; RV64M-NEXT: mul a6, a0, a3
+; RV64M-NEXT: mulhu a2, a1, a2
+; RV64M-NEXT: mulhu a0, a0, a3
+; RV64M-NEXT: mul a7, a1, a3
+; RV64M-NEXT: add a4, a5, a4
+; RV64M-NEXT: mulhu a1, a1, a3
+; RV64M-NEXT: add a3, a6, a4
+; RV64M-NEXT: sltu a4, a4, a5
+; RV64M-NEXT: sltu a3, a3, a6
+; RV64M-NEXT: add a2, a2, a4
+; RV64M-NEXT: add a0, a0, a3
+; RV64M-NEXT: add a3, a2, a0
+; RV64M-NEXT: add a0, a7, a3
+; RV64M-NEXT: sltu a2, a3, a2
+; RV64M-NEXT: sltu a3, a0, a7
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: add a1, a1, a3
+; RV64M-NEXT: ret
+ %r = call i96 @llvm.umulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @umulh_i128(i128 %x, i128 %y) nounwind {
+; RV32I-LABEL: umulh_i128:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -96
+; RV32I-NEXT: sw ra, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s8, 0(a2)
+; RV32I-NEXT: lw s7, 4(a2)
+; RV32I-NEXT: lw a3, 8(a2)
+; RV32I-NEXT: sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a2, 12(a2)
+; RV32I-NEXT: sw a2, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s6, 0(a1)
+; RV32I-NEXT: lw s4, 4(a1)
+; RV32I-NEXT: lw s3, 8(a1)
+; RV32I-NEXT: lw s2, 12(a1)
+; RV32I-NEXT: sw a0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: sltu a0, s9, a0
+; RV32I-NEXT: add s0, a1, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: sltu a0, s9, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, s0, a0
+; RV32I-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s4
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s1
+; RV32I-NEXT: sltu a2, s1, s0
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s0, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s10, a0
+; RV32I-NEXT: mv s11, a1
+; RV32I-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: add s11, a0, s11
+; RV32I-NEXT: sw s3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, s10, s4
+; RV32I-NEXT: add a2, a0, s11
+; RV32I-NEXT: add s5, a2, s0
+; RV32I-NEXT: sltu s0, s4, s10
+; RV32I-NEXT: add s5, s5, s0
+; RV32I-NEXT: beq s5, a2, .LBB4_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu s0, s5, a2
+; RV32I-NEXT: .LBB4_2:
+; RV32I-NEXT: sltu a3, s11, s9
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a3, s8, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s2, a3, a0
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s7
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s8, a0
+; RV32I-NEXT: sw a1, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s1, a0, s2
+; RV32I-NEXT: add s7, s1, s0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s11, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw a0, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: add s10, a0, s0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, s11, s4
+; RV32I-NEXT: add a2, a0, s10
+; RV32I-NEXT: add a3, a2, s5
+; RV32I-NEXT: sltu s0, s4, s11
+; RV32I-NEXT: add a3, a3, s0
+; RV32I-NEXT: beq a3, a2, .LBB4_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu s0, a3, a2
+; RV32I-NEXT: .LBB4_4:
+; RV32I-NEXT: sltu a3, s10, s3
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s9, s9, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s9, a0
+; RV32I-NEXT: lw a0, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s10, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s7, s1
+; RV32I-NEXT: add a3, a0, s3
+; RV32I-NEXT: sltu a4, s1, s8
+; RV32I-NEXT: add s0, a3, s0
+; RV32I-NEXT: lw a5, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sltu a5, s2, a5
+; RV32I-NEXT: add s2, s7, s0
+; RV32I-NEXT: lw a6, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a5, a6, a5
+; RV32I-NEXT: add a2, a4, a2
+; RV32I-NEXT: sltu a4, s3, s9
+; RV32I-NEXT: sltu a0, a3, a0
+; RV32I-NEXT: add a1, a1, a4
+; RV32I-NEXT: sltu a3, s0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, a5, a2
+; RV32I-NEXT: add a0, a0, a3
+; RV32I-NEXT: sltu s1, s2, s7
+; RV32I-NEXT: add a0, s3, a0
+; RV32I-NEXT: add s7, a0, s1
+; RV32I-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s4, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: add s8, a0, s6
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s10
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, s0, s2
+; RV32I-NEXT: add a2, a0, s8
+; RV32I-NEXT: sltu s6, s2, s0
+; RV32I-NEXT: add s0, a2, s6
+; RV32I-NEXT: add s0, s0, s7
+; RV32I-NEXT: beq s0, a2, .LBB4_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu s6, s0, a2
+; RV32I-NEXT: .LBB4_6:
+; RV32I-NEXT: beq s7, s3, .LBB4_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s1, s7, s3
+; RV32I-NEXT: .LBB4_8:
+; RV32I-NEXT: sltu a3, s8, s5
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s4, s4, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s4, a0
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s3, s4
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s1, s3, s1
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add s6, s1, s6
+; RV32I-NEXT: sltu a2, s1, s3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sltu a1, s6, s1
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lw a1, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw s2, 0(a1)
+; RV32I-NEXT: sw s0, 4(a1)
+; RV32I-NEXT: sw s6, 8(a1)
+; RV32I-NEXT: sw a0, 12(a1)
+; RV32I-NEXT: lw ra, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 96
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_i128:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -48
+; RV64I-NEXT: sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a3
+; RV64I-NEXT: mv s1, a2
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: mv s3, a0
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s4, a1
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s4, a0, s4
+; RV64I-NEXT: sltu a0, s4, a0
+; RV64I-NEXT: add s1, a1, a0
+; RV64I-NEXT: mv a0, s3
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s4, a0, s4
+; RV64I-NEXT: sltu a0, s4, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s3, s1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add a2, a0, s3
+; RV64I-NEXT: sltu a3, s3, s1
+; RV64I-NEXT: sltu a0, a2, a0
+; RV64I-NEXT: add a1, a1, a3
+; RV64I-NEXT: add a1, a1, a0
+; RV64I-NEXT: mv a0, a2
+; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 48
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: umulh_i128:
+; RV32M: # %bb.0:
+; RV32M-NEXT: addi sp, sp, -32
+; RV32M-NEXT: sw s0, 28(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s1, 24(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s2, 20(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s3, 16(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s4, 12(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s5, 8(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s6, 4(sp) # 4-byte Folded Spill
+; RV32M-NEXT: lw a7, 0(a1)
+; RV32M-NEXT: lw a5, 4(a1)
+; RV32M-NEXT: lw t0, 0(a2)
+; RV32M-NEXT: lw a6, 4(a2)
+; RV32M-NEXT: lw a4, 8(a2)
+; RV32M-NEXT: lw a2, 12(a2)
+; RV32M-NEXT: lw a3, 8(a1)
+; RV32M-NEXT: lw a1, 12(a1)
+; RV32M-NEXT: mulhu t3, a7, t0
+; RV32M-NEXT: mul t4, a5, t0
+; RV32M-NEXT: mul t5, a7, a6
+; RV32M-NEXT: mulhu t6, a5, t0
+; RV32M-NEXT: mulhu s0, a7, a6
+; RV32M-NEXT: mul s1, a5, a6
+; RV32M-NEXT: mulhu s2, a5, a6
+; RV32M-NEXT: mulhu s3, a3, t0
+; RV32M-NEXT: mul t1, a1, t0
+; RV32M-NEXT: mul t2, a3, a6
+; RV32M-NEXT: add t3, t4, t3
+; RV32M-NEXT: sltu t4, t3, t4
+; RV32M-NEXT: add t3, t5, t3
+; RV32M-NEXT: sltu t3, t3, t5
+; RV32M-NEXT: mul t5, a3, t0
+; RV32M-NEXT: add t4, t6, t4
+; RV32M-NEXT: add t3, s0, t3
+; RV32M-NEXT: add t3, t4, t3
+; RV32M-NEXT: sltu t4, t3, t4
+; RV32M-NEXT: add t3, s1, t3
+; RV32M-NEXT: add t4, s2, t4
+; RV32M-NEXT: sltu t6, t3, s1
+; RV32M-NEXT: add s0, t1, s3
+; RV32M-NEXT: add t6, t4, t6
+; RV32M-NEXT: add s1, t2, s0
+; RV32M-NEXT: add t4, t5, t3
+; RV32M-NEXT: add t6, s1, t6
+; RV32M-NEXT: sltu t3, t4, t5
+; RV32M-NEXT: add t6, t6, t3
+; RV32M-NEXT: beq t6, s1, .LBB4_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu t3, t6, s1
+; RV32M-NEXT: .LBB4_2:
+; RV32M-NEXT: mulhu t0, a1, t0
+; RV32M-NEXT: mulhu s4, a3, a6
+; RV32M-NEXT: mulhu s5, a7, a4
+; RV32M-NEXT: mul s2, a5, a4
+; RV32M-NEXT: mul s3, a7, a2
+; RV32M-NEXT: mul s6, a7, a4
+; RV32M-NEXT: mul t5, a1, a6
+; RV32M-NEXT: sltu t1, s0, t1
+; RV32M-NEXT: sltu s0, s1, t2
+; RV32M-NEXT: add t2, t0, t1
+; RV32M-NEXT: add s0, s4, s0
+; RV32M-NEXT: add s1, s2, s5
+; RV32M-NEXT: add s0, t2, s0
+; RV32M-NEXT: add s4, s3, s1
+; RV32M-NEXT: add t4, s6, t4
+; RV32M-NEXT: add t6, s4, t6
+; RV32M-NEXT: sltu t1, t4, s6
+; RV32M-NEXT: add t4, t5, s0
+; RV32M-NEXT: add t6, t6, t1
+; RV32M-NEXT: add t0, t4, t3
+; RV32M-NEXT: beq t6, s4, .LBB4_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu t1, t6, s4
+; RV32M-NEXT: .LBB4_4:
+; RV32M-NEXT: mulhu t3, a5, a4
+; RV32M-NEXT: mulhu a7, a7, a2
+; RV32M-NEXT: sltu t6, s1, s2
+; RV32M-NEXT: sltu s1, s4, s3
+; RV32M-NEXT: mul s2, a5, a2
+; RV32M-NEXT: sltu s3, t0, t4
+; RV32M-NEXT: sltu t4, t4, t5
+; RV32M-NEXT: sltu t2, s0, t2
+; RV32M-NEXT: mulhu t5, a1, a6
+; RV32M-NEXT: mulhu a6, a5, a2
+; RV32M-NEXT: mulhu s0, a3, a4
+; RV32M-NEXT: add t3, t3, t6
+; RV32M-NEXT: add a7, a7, s1
+; RV32M-NEXT: add t2, t2, t4
+; RV32M-NEXT: add a7, t3, a7
+; RV32M-NEXT: add t2, t2, s3
+; RV32M-NEXT: add t4, s2, a7
+; RV32M-NEXT: sltu a7, a7, t3
+; RV32M-NEXT: mul a5, a1, a4
+; RV32M-NEXT: add t3, t4, t1
+; RV32M-NEXT: mul t6, a3, a4
+; RV32M-NEXT: add s1, t0, t3
+; RV32M-NEXT: mul t1, a3, a2
+; RV32M-NEXT: sltu s2, t4, s2
+; RV32M-NEXT: add a6, a6, a7
+; RV32M-NEXT: sltu a7, t3, t4
+; RV32M-NEXT: add a6, a6, s2
+; RV32M-NEXT: add t5, t5, t2
+; RV32M-NEXT: add a6, a6, a7
+; RV32M-NEXT: sltu t0, s1, t0
+; RV32M-NEXT: add a7, t5, a6
+; RV32M-NEXT: add t3, a5, s0
+; RV32M-NEXT: add a6, t6, s1
+; RV32M-NEXT: add t4, t1, t3
+; RV32M-NEXT: sltu t2, a6, t6
+; RV32M-NEXT: add t6, a7, t0
+; RV32M-NEXT: add a7, t4, t2
+; RV32M-NEXT: add a7, a7, t6
+; RV32M-NEXT: beq a7, t4, .LBB4_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu t2, a7, t4
+; RV32M-NEXT: .LBB4_6:
+; RV32M-NEXT: beq t6, t5, .LBB4_8
+; RV32M-NEXT: # %bb.7:
+; RV32M-NEXT: sltu t0, t6, t5
+; RV32M-NEXT: .LBB4_8:
+; RV32M-NEXT: mulhu a4, a1, a4
+; RV32M-NEXT: mulhu a3, a3, a2
+; RV32M-NEXT: mul t5, a1, a2
+; RV32M-NEXT: mulhu a1, a1, a2
+; RV32M-NEXT: sltu a2, t3, a5
+; RV32M-NEXT: sltu a5, t4, t1
+; RV32M-NEXT: add a2, a4, a2
+; RV32M-NEXT: add a3, a3, a5
+; RV32M-NEXT: add a3, a2, a3
+; RV32M-NEXT: add a4, t5, a3
+; RV32M-NEXT: sltu a2, a3, a2
+; RV32M-NEXT: add t0, a4, t0
+; RV32M-NEXT: sltu a3, a4, t5
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add t2, t0, t2
+; RV32M-NEXT: sltu a2, t0, a4
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: sltu a3, t2, t0
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: sw a6, 0(a0)
+; RV32M-NEXT: sw a7, 4(a0)
+; RV32M-NEXT: sw t2, 8(a0)
+; RV32M-NEXT: sw a1, 12(a0)
+; RV32M-NEXT: lw s0, 28(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s1, 24(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s2, 20(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s3, 16(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s4, 12(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s5, 8(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s6, 4(sp) # 4-byte Folded Reload
+; RV32M-NEXT: addi sp, sp, 32
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: umulh_i128:
+; RV64M: # %bb.0:
+; RV64M-NEXT: mulhu a4, a0, a2
+; RV64M-NEXT: mul a5, a1, a2
+; RV64M-NEXT: mul a6, a0, a3
+; RV64M-NEXT: mulhu a2, a1, a2
+; RV64M-NEXT: mulhu a0, a0, a3
+; RV64M-NEXT: mul a7, a1, a3
+; RV64M-NEXT: add a4, a5, a4
+; RV64M-NEXT: mulhu a1, a1, a3
+; RV64M-NEXT: add a3, a6, a4
+; RV64M-NEXT: sltu a4, a4, a5
+; RV64M-NEXT: sltu a3, a3, a6
+; RV64M-NEXT: add a2, a2, a4
+; RV64M-NEXT: add a0, a0, a3
+; RV64M-NEXT: add a3, a2, a0
+; RV64M-NEXT: add a0, a7, a3
+; RV64M-NEXT: sltu a2, a3, a2
+; RV64M-NEXT: sltu a3, a0, a7
+; RV64M-NEXT: add a1, a1, a2
+; RV64M-NEXT: add a1, a1, a3
+; RV64M-NEXT: ret
+ %r = call i128 @llvm.umulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @umulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; RV32I-LABEL: umulh_v8i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu s6, 16(a1)
+; RV32I-NEXT: lbu s5, 20(a1)
+; RV32I-NEXT: lbu a3, 24(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 28(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu s7, 16(a2)
+; RV32I-NEXT: lbu s8, 20(a2)
+; RV32I-NEXT: lbu a3, 24(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 28(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 0(a1)
+; RV32I-NEXT: lbu s9, 4(a1)
+; RV32I-NEXT: lbu s10, 8(a1)
+; RV32I-NEXT: lbu s11, 12(a1)
+; RV32I-NEXT: lbu a1, 0(a2)
+; RV32I-NEXT: lbu s1, 4(a2)
+; RV32I-NEXT: lbu s2, 8(a2)
+; RV32I-NEXT: lbu s3, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 8
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 8
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 8
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 8
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 8
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s5, a0, 8
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 8
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sb s6, 4(s0)
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sb s5, 5(s0)
+; RV32I-NEXT: sb s7, 6(s0)
+; RV32I-NEXT: sb a0, 7(s0)
+; RV32I-NEXT: sb s4, 0(s0)
+; RV32I-NEXT: sb s1, 1(s0)
+; RV32I-NEXT: sb s2, 2(s0)
+; RV32I-NEXT: sb s3, 3(s0)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v8i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -144
+; RV64I-NEXT: sd ra, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu s6, 32(a1)
+; RV64I-NEXT: lbu s5, 40(a1)
+; RV64I-NEXT: lbu a3, 48(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 56(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu s7, 32(a2)
+; RV64I-NEXT: lbu s8, 40(a2)
+; RV64I-NEXT: lbu a3, 48(a2)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 56(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 0(a1)
+; RV64I-NEXT: lbu s9, 8(a1)
+; RV64I-NEXT: lbu s10, 16(a1)
+; RV64I-NEXT: lbu s11, 24(a1)
+; RV64I-NEXT: lbu a1, 0(a2)
+; RV64I-NEXT: lbu s1, 8(a2)
+; RV64I-NEXT: lbu s2, 16(a2)
+; RV64I-NEXT: lbu s3, 24(a2)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 8
+; RV64I-NEXT: mv a0, s9
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 8
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 8
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 8
+; RV64I-NEXT: mv a0, s6
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 8
+; RV64I-NEXT: mv a0, s5
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 8
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 8
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sb s6, 4(s0)
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sb s5, 5(s0)
+; RV64I-NEXT: sb s7, 6(s0)
+; RV64I-NEXT: sb a0, 7(s0)
+; RV64I-NEXT: sb s4, 0(s0)
+; RV64I-NEXT: sb s1, 1(s0)
+; RV64I-NEXT: sb s2, 2(s0)
+; RV64I-NEXT: sb s3, 3(s0)
+; RV64I-NEXT: ld ra, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 144
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v8i8:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -16
+; RV32IM-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lbu a3, 16(a2)
+; RV32IM-NEXT: lbu a4, 20(a2)
+; RV32IM-NEXT: lbu a5, 24(a2)
+; RV32IM-NEXT: lbu a6, 28(a2)
+; RV32IM-NEXT: lbu a7, 0(a2)
+; RV32IM-NEXT: lbu t0, 4(a2)
+; RV32IM-NEXT: lbu t1, 8(a2)
+; RV32IM-NEXT: lbu a2, 12(a2)
+; RV32IM-NEXT: lbu t2, 0(a1)
+; RV32IM-NEXT: lbu t3, 4(a1)
+; RV32IM-NEXT: lbu t4, 8(a1)
+; RV32IM-NEXT: lbu t5, 12(a1)
+; RV32IM-NEXT: lbu t6, 16(a1)
+; RV32IM-NEXT: lbu s0, 20(a1)
+; RV32IM-NEXT: lbu s1, 24(a1)
+; RV32IM-NEXT: lbu a1, 28(a1)
+; RV32IM-NEXT: mul a7, t2, a7
+; RV32IM-NEXT: mul t0, t3, t0
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: mul a2, t5, a2
+; RV32IM-NEXT: mul a3, t6, a3
+; RV32IM-NEXT: mul a4, s0, a4
+; RV32IM-NEXT: mul a5, s1, a5
+; RV32IM-NEXT: mul a1, a1, a6
+; RV32IM-NEXT: srli a6, a7, 8
+; RV32IM-NEXT: srli a7, t0, 8
+; RV32IM-NEXT: srli t0, t1, 8
+; RV32IM-NEXT: srli a2, a2, 8
+; RV32IM-NEXT: srli a3, a3, 8
+; RV32IM-NEXT: srli a4, a4, 8
+; RV32IM-NEXT: srli a5, a5, 8
+; RV32IM-NEXT: srli a1, a1, 8
+; RV32IM-NEXT: sb a3, 4(a0)
+; RV32IM-NEXT: sb a4, 5(a0)
+; RV32IM-NEXT: sb a5, 6(a0)
+; RV32IM-NEXT: sb a1, 7(a0)
+; RV32IM-NEXT: sb a6, 0(a0)
+; RV32IM-NEXT: sb a7, 1(a0)
+; RV32IM-NEXT: sb t0, 2(a0)
+; RV32IM-NEXT: sb a2, 3(a0)
+; RV32IM-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 16
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v8i8:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -16
+; RV64IM-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lbu a3, 32(a2)
+; RV64IM-NEXT: lbu a4, 40(a2)
+; RV64IM-NEXT: lbu a5, 48(a2)
+; RV64IM-NEXT: lbu a6, 56(a2)
+; RV64IM-NEXT: lbu a7, 0(a2)
+; RV64IM-NEXT: lbu t0, 8(a2)
+; RV64IM-NEXT: lbu t1, 16(a2)
+; RV64IM-NEXT: lbu a2, 24(a2)
+; RV64IM-NEXT: lbu t2, 0(a1)
+; RV64IM-NEXT: lbu t3, 8(a1)
+; RV64IM-NEXT: lbu t4, 16(a1)
+; RV64IM-NEXT: lbu t5, 24(a1)
+; RV64IM-NEXT: lbu t6, 32(a1)
+; RV64IM-NEXT: lbu s0, 40(a1)
+; RV64IM-NEXT: lbu s1, 48(a1)
+; RV64IM-NEXT: lbu a1, 56(a1)
+; RV64IM-NEXT: mul a7, t2, a7
+; RV64IM-NEXT: mul t0, t3, t0
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: mul a2, t5, a2
+; RV64IM-NEXT: mul a3, t6, a3
+; RV64IM-NEXT: mul a4, s0, a4
+; RV64IM-NEXT: mul a5, s1, a5
+; RV64IM-NEXT: mul a1, a1, a6
+; RV64IM-NEXT: srli a6, a7, 8
+; RV64IM-NEXT: srli a7, t0, 8
+; RV64IM-NEXT: srli t0, t1, 8
+; RV64IM-NEXT: srli a2, a2, 8
+; RV64IM-NEXT: srli a3, a3, 8
+; RV64IM-NEXT: srli a4, a4, 8
+; RV64IM-NEXT: srli a5, a5, 8
+; RV64IM-NEXT: srli a1, a1, 8
+; RV64IM-NEXT: sb a3, 4(a0)
+; RV64IM-NEXT: sb a4, 5(a0)
+; RV64IM-NEXT: sb a5, 6(a0)
+; RV64IM-NEXT: sb a1, 7(a0)
+; RV64IM-NEXT: sb a6, 0(a0)
+; RV64IM-NEXT: sb a7, 1(a0)
+; RV64IM-NEXT: sb t0, 2(a0)
+; RV64IM-NEXT: sb a2, 3(a0)
+; RV64IM-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 16
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v8i8:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <8 x i8> @llvm.umulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @umulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; RV32I-LABEL: umulh_v4i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu a3, 0(a1)
+; RV32I-NEXT: lhu s0, 4(a1)
+; RV32I-NEXT: lhu s1, 8(a1)
+; RV32I-NEXT: lhu s2, 12(a1)
+; RV32I-NEXT: lhu a1, 0(a2)
+; RV32I-NEXT: lhu s3, 4(a2)
+; RV32I-NEXT: lhu s4, 8(a2)
+; RV32I-NEXT: lhu s5, 12(a2)
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 16
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s0, a0, 16
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 16
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s5
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sh s7, 0(s6)
+; RV32I-NEXT: srli a0, a0, 16
+; RV32I-NEXT: sh s0, 2(s6)
+; RV32I-NEXT: sh s1, 4(s6)
+; RV32I-NEXT: sh a0, 6(s6)
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v4i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu a3, 0(a1)
+; RV64I-NEXT: lhu s0, 8(a1)
+; RV64I-NEXT: lhu s1, 16(a1)
+; RV64I-NEXT: lhu s2, 24(a1)
+; RV64I-NEXT: lhu a1, 0(a2)
+; RV64I-NEXT: lhu s3, 8(a2)
+; RV64I-NEXT: lhu s4, 16(a2)
+; RV64I-NEXT: lhu s5, 24(a2)
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 16
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 16
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 16
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sh s7, 0(s6)
+; RV64I-NEXT: srli a0, a0, 16
+; RV64I-NEXT: sh s0, 2(s6)
+; RV64I-NEXT: sh s1, 4(s6)
+; RV64I-NEXT: sh a0, 6(s6)
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v4i16:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lhu a3, 0(a2)
+; RV32IM-NEXT: lhu a4, 4(a2)
+; RV32IM-NEXT: lhu a5, 8(a2)
+; RV32IM-NEXT: lhu a2, 12(a2)
+; RV32IM-NEXT: lhu a6, 0(a1)
+; RV32IM-NEXT: lhu a7, 4(a1)
+; RV32IM-NEXT: lhu t0, 8(a1)
+; RV32IM-NEXT: lhu a1, 12(a1)
+; RV32IM-NEXT: mul a3, a6, a3
+; RV32IM-NEXT: mul a4, a7, a4
+; RV32IM-NEXT: mul a5, t0, a5
+; RV32IM-NEXT: mul a1, a1, a2
+; RV32IM-NEXT: srli a3, a3, 16
+; RV32IM-NEXT: srli a4, a4, 16
+; RV32IM-NEXT: srli a5, a5, 16
+; RV32IM-NEXT: srli a1, a1, 16
+; RV32IM-NEXT: sh a3, 0(a0)
+; RV32IM-NEXT: sh a4, 2(a0)
+; RV32IM-NEXT: sh a5, 4(a0)
+; RV32IM-NEXT: sh a1, 6(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v4i16:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lhu a3, 0(a2)
+; RV64IM-NEXT: lhu a4, 8(a2)
+; RV64IM-NEXT: lhu a5, 16(a2)
+; RV64IM-NEXT: lhu a2, 24(a2)
+; RV64IM-NEXT: lhu a6, 0(a1)
+; RV64IM-NEXT: lhu a7, 8(a1)
+; RV64IM-NEXT: lhu t0, 16(a1)
+; RV64IM-NEXT: lhu a1, 24(a1)
+; RV64IM-NEXT: mul a3, a6, a3
+; RV64IM-NEXT: mul a4, a7, a4
+; RV64IM-NEXT: mul a5, t0, a5
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 16
+; RV64IM-NEXT: srli a4, a4, 16
+; RV64IM-NEXT: srli a5, a5, 16
+; RV64IM-NEXT: srli a1, a1, 16
+; RV64IM-NEXT: sh a3, 0(a0)
+; RV64IM-NEXT: sh a4, 2(a0)
+; RV64IM-NEXT: sh a5, 4(a0)
+; RV64IM-NEXT: sh a1, 6(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v4i16:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <4 x i16> @llvm.umulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @umulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; RV32I-LABEL: umulh_v2i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -16
+; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 16
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v2i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: slli a1, a1, 32
+; RV64I-NEXT: slli a3, a3, 32
+; RV64I-NEXT: srli s0, a1, 32
+; RV64I-NEXT: srli s1, a3, 32
+; RV64I-NEXT: slli a0, a0, 32
+; RV64I-NEXT: slli a1, a2, 32
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: srli a1, a1, 32
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: srli a1, a1, 32
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v2i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: mulhu a0, a0, a2
+; RV32IM-NEXT: mulhu a1, a1, a3
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v2i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: slli a2, a2, 32
+; RV64IM-NEXT: slli a0, a0, 32
+; RV64IM-NEXT: slli a3, a3, 32
+; RV64IM-NEXT: slli a1, a1, 32
+; RV64IM-NEXT: mulhu a0, a0, a2
+; RV64IM-NEXT: mulhu a1, a1, a3
+; RV64IM-NEXT: srli a0, a0, 32
+; RV64IM-NEXT: srli a1, a1, 32
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v2i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <2 x i32> @llvm.umulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @umulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; RV32I-LABEL: umulh_v1i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv s0, a3
+; RV32I-NEXT: mv s1, a2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: mv s3, a0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s4
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add s1, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s4, a0, s4
+; RV32I-NEXT: sltu a0, s4, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s1, a0
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: sltu a3, s3, s1
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a1, a1, a3
+; RV32I-NEXT: add a1, a1, a0
+; RV32I-NEXT: mv a0, a2
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v1i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -16
+; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a2, a1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, a1
+; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 16
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v1i64:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: mulhu a4, a0, a2
+; RV32IM-NEXT: mul a5, a1, a2
+; RV32IM-NEXT: mul a6, a0, a3
+; RV32IM-NEXT: mulhu a2, a1, a2
+; RV32IM-NEXT: mulhu a0, a0, a3
+; RV32IM-NEXT: mul a7, a1, a3
+; RV32IM-NEXT: add a4, a5, a4
+; RV32IM-NEXT: mulhu a1, a1, a3
+; RV32IM-NEXT: add a3, a6, a4
+; RV32IM-NEXT: sltu a4, a4, a5
+; RV32IM-NEXT: sltu a3, a3, a6
+; RV32IM-NEXT: add a2, a2, a4
+; RV32IM-NEXT: add a0, a0, a3
+; RV32IM-NEXT: add a3, a2, a0
+; RV32IM-NEXT: add a0, a7, a3
+; RV32IM-NEXT: sltu a2, a3, a2
+; RV32IM-NEXT: sltu a3, a0, a7
+; RV32IM-NEXT: add a1, a1, a2
+; RV32IM-NEXT: add a1, a1, a3
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v1i64:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: mulhu a0, a0, a1
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v1i64:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <1 x i64> @llvm.umulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @umulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; RV32I-LABEL: umulh_v16i8:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -144
+; RV32I-NEXT: sw ra, 140(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 136(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 132(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 128(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 120(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 116(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 112(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 104(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 100(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 96(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 48(a1)
+; RV32I-NEXT: sw a3, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 52(a1)
+; RV32I-NEXT: sw a3, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 56(a1)
+; RV32I-NEXT: sw a3, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 60(a1)
+; RV32I-NEXT: sw a3, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 48(a2)
+; RV32I-NEXT: sw a3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 52(a2)
+; RV32I-NEXT: sw a3, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 56(a2)
+; RV32I-NEXT: sw a3, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 60(a2)
+; RV32I-NEXT: sw a3, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 32(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 36(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 40(a1)
+; RV32I-NEXT: sw a3, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 44(a1)
+; RV32I-NEXT: sw a3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 32(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 36(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 40(a2)
+; RV32I-NEXT: sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 44(a2)
+; RV32I-NEXT: sw a3, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu s10, 16(a1)
+; RV32I-NEXT: lbu a3, 20(a1)
+; RV32I-NEXT: sw a3, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 24(a1)
+; RV32I-NEXT: sw a3, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 28(a1)
+; RV32I-NEXT: sw a3, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lbu a3, 0(a1)
+; RV32I-NEXT: lbu s11, 4(a1)
+; RV32I-NEXT: lbu s0, 8(a1)
+; RV32I-NEXT: lbu s1, 12(a1)
+; RV32I-NEXT: lbu s2, 16(a2)
+; RV32I-NEXT: lbu s3, 20(a2)
+; RV32I-NEXT: lbu s4, 24(a2)
+; RV32I-NEXT: lbu s9, 28(a2)
+; RV32I-NEXT: lbu a1, 0(a2)
+; RV32I-NEXT: lbu s6, 4(a2)
+; RV32I-NEXT: lbu s7, 8(a2)
+; RV32I-NEXT: lbu s8, 12(a2)
+; RV32I-NEXT: mv s5, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s6
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sw a0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 8
+; RV32I-NEXT: lw a0, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 8
+; RV32I-NEXT: lw a0, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s4
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 8
+; RV32I-NEXT: lw a0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a1, s9
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 8
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s8, a0, 8
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s9, a0, 8
+; RV32I-NEXT: lw a0, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s10, a0, 8
+; RV32I-NEXT: lw a0, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s11, a0, 8
+; RV32I-NEXT: lw a0, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 8
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s0, a0, 8
+; RV32I-NEXT: lw a0, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 8
+; RV32I-NEXT: lw a0, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sb s6, 12(s5)
+; RV32I-NEXT: srli a0, a0, 8
+; RV32I-NEXT: sb s0, 13(s5)
+; RV32I-NEXT: sb s1, 14(s5)
+; RV32I-NEXT: sb a0, 15(s5)
+; RV32I-NEXT: sb s8, 8(s5)
+; RV32I-NEXT: sb s9, 9(s5)
+; RV32I-NEXT: sb s10, 10(s5)
+; RV32I-NEXT: sb s11, 11(s5)
+; RV32I-NEXT: sb s2, 4(s5)
+; RV32I-NEXT: sb s3, 5(s5)
+; RV32I-NEXT: sb s4, 6(s5)
+; RV32I-NEXT: sb s7, 7(s5)
+; RV32I-NEXT: lw a0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 0(s5)
+; RV32I-NEXT: lw a0, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 1(s5)
+; RV32I-NEXT: lw a0, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 2(s5)
+; RV32I-NEXT: lw a0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sb a0, 3(s5)
+; RV32I-NEXT: lw ra, 140(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 136(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 132(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 128(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 124(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 116(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 112(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 108(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 100(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 96(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 144
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v16i8:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -288
+; RV64I-NEXT: sd ra, 280(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 272(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 264(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 256(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 248(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 240(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 232(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 224(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 216(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 208(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 200(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 192(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 184(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 96(a1)
+; RV64I-NEXT: sd a3, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 104(a1)
+; RV64I-NEXT: sd a3, 144(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 112(a1)
+; RV64I-NEXT: sd a3, 160(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 120(a1)
+; RV64I-NEXT: sd a3, 176(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 96(a2)
+; RV64I-NEXT: sd a3, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 104(a2)
+; RV64I-NEXT: sd a3, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 112(a2)
+; RV64I-NEXT: sd a3, 152(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 120(a2)
+; RV64I-NEXT: sd a3, 168(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 64(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 72(a1)
+; RV64I-NEXT: sd a3, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 80(a1)
+; RV64I-NEXT: sd a3, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 88(a1)
+; RV64I-NEXT: sd a3, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 64(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 72(a2)
+; RV64I-NEXT: sd a3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 80(a2)
+; RV64I-NEXT: sd a3, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 88(a2)
+; RV64I-NEXT: sd a3, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu s10, 32(a1)
+; RV64I-NEXT: lbu a3, 40(a1)
+; RV64I-NEXT: sd a3, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 48(a1)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 56(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lbu a3, 0(a1)
+; RV64I-NEXT: lbu s11, 8(a1)
+; RV64I-NEXT: lbu s0, 16(a1)
+; RV64I-NEXT: lbu s1, 24(a1)
+; RV64I-NEXT: lbu s2, 32(a2)
+; RV64I-NEXT: lbu s3, 40(a2)
+; RV64I-NEXT: lbu s4, 48(a2)
+; RV64I-NEXT: lbu s9, 56(a2)
+; RV64I-NEXT: lbu a1, 0(a2)
+; RV64I-NEXT: lbu s6, 8(a2)
+; RV64I-NEXT: lbu s7, 16(a2)
+; RV64I-NEXT: lbu s8, 24(a2)
+; RV64I-NEXT: mv s5, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s6
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sd a0, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 8
+; RV64I-NEXT: ld a0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 8
+; RV64I-NEXT: ld a0, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 8
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: mv a1, s9
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 8
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s8, a0, 8
+; RV64I-NEXT: ld a0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s9, a0, 8
+; RV64I-NEXT: ld a0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s10, a0, 8
+; RV64I-NEXT: ld a0, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s11, a0, 8
+; RV64I-NEXT: ld a0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 8
+; RV64I-NEXT: ld a0, 144(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 8
+; RV64I-NEXT: ld a0, 160(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 152(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 8
+; RV64I-NEXT: ld a0, 176(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 168(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sb s6, 12(s5)
+; RV64I-NEXT: srli a0, a0, 8
+; RV64I-NEXT: sb s0, 13(s5)
+; RV64I-NEXT: sb s1, 14(s5)
+; RV64I-NEXT: sb a0, 15(s5)
+; RV64I-NEXT: sb s8, 8(s5)
+; RV64I-NEXT: sb s9, 9(s5)
+; RV64I-NEXT: sb s10, 10(s5)
+; RV64I-NEXT: sb s11, 11(s5)
+; RV64I-NEXT: sb s2, 4(s5)
+; RV64I-NEXT: sb s3, 5(s5)
+; RV64I-NEXT: sb s4, 6(s5)
+; RV64I-NEXT: sb s7, 7(s5)
+; RV64I-NEXT: ld a0, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 0(s5)
+; RV64I-NEXT: ld a0, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 1(s5)
+; RV64I-NEXT: ld a0, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 2(s5)
+; RV64I-NEXT: ld a0, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: sb a0, 3(s5)
+; RV64I-NEXT: ld ra, 280(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 272(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 264(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 256(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 248(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 240(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 232(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 224(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 216(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 208(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 200(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 192(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 288
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v16i8:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -48
+; RV32IM-NEXT: sw s0, 44(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 40(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s3, 32(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s5, 24(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s6, 20(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s7, 16(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s8, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s9, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s10, 4(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s11, 0(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lbu t3, 48(a2)
+; RV32IM-NEXT: lbu t2, 52(a2)
+; RV32IM-NEXT: lbu t1, 56(a2)
+; RV32IM-NEXT: lbu a4, 60(a2)
+; RV32IM-NEXT: lbu t6, 48(a1)
+; RV32IM-NEXT: lbu t5, 52(a1)
+; RV32IM-NEXT: lbu t4, 56(a1)
+; RV32IM-NEXT: lbu t0, 60(a1)
+; RV32IM-NEXT: lbu s2, 16(a2)
+; RV32IM-NEXT: lbu s3, 20(a2)
+; RV32IM-NEXT: lbu s1, 24(a2)
+; RV32IM-NEXT: lbu s0, 28(a2)
+; RV32IM-NEXT: lbu a3, 0(a2)
+; RV32IM-NEXT: lbu a5, 0(a1)
+; RV32IM-NEXT: lbu a6, 4(a1)
+; RV32IM-NEXT: lbu a7, 4(a2)
+; RV32IM-NEXT: lbu s4, 8(a2)
+; RV32IM-NEXT: lbu s5, 12(a2)
+; RV32IM-NEXT: lbu s6, 8(a1)
+; RV32IM-NEXT: lbu s7, 12(a1)
+; RV32IM-NEXT: lbu s8, 16(a1)
+; RV32IM-NEXT: mul a3, a5, a3
+; RV32IM-NEXT: mul a5, a6, a7
+; RV32IM-NEXT: lbu s9, 20(a1)
+; RV32IM-NEXT: lbu s10, 24(a1)
+; RV32IM-NEXT: lbu s11, 28(a1)
+; RV32IM-NEXT: mul a6, s6, s4
+; RV32IM-NEXT: mul a7, s7, s5
+; RV32IM-NEXT: mul s2, s8, s2
+; RV32IM-NEXT: lbu s4, 32(a2)
+; RV32IM-NEXT: lbu s5, 36(a2)
+; RV32IM-NEXT: lbu s6, 40(a2)
+; RV32IM-NEXT: lbu a2, 44(a2)
+; RV32IM-NEXT: mul t3, t6, t3
+; RV32IM-NEXT: mul t2, t5, t2
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: lbu t4, 32(a1)
+; RV32IM-NEXT: lbu t5, 36(a1)
+; RV32IM-NEXT: lbu t6, 40(a1)
+; RV32IM-NEXT: lbu a1, 44(a1)
+; RV32IM-NEXT: mul s3, s9, s3
+; RV32IM-NEXT: mul a4, t0, a4
+; RV32IM-NEXT: mul t0, s10, s1
+; RV32IM-NEXT: mul s0, s11, s0
+; RV32IM-NEXT: mul t4, t4, s4
+; RV32IM-NEXT: mul t5, t5, s5
+; RV32IM-NEXT: mul t6, t6, s6
+; RV32IM-NEXT: mul a1, a1, a2
+; RV32IM-NEXT: srli a2, t3, 8
+; RV32IM-NEXT: srli t2, t2, 8
+; RV32IM-NEXT: sb a2, 12(a0)
+; RV32IM-NEXT: srli a2, t1, 8
+; RV32IM-NEXT: sb t2, 13(a0)
+; RV32IM-NEXT: srli a4, a4, 8
+; RV32IM-NEXT: sb a2, 14(a0)
+; RV32IM-NEXT: sb a4, 15(a0)
+; RV32IM-NEXT: srli a2, t4, 8
+; RV32IM-NEXT: srli a4, t5, 8
+; RV32IM-NEXT: sb a2, 8(a0)
+; RV32IM-NEXT: srli a2, t6, 8
+; RV32IM-NEXT: sb a4, 9(a0)
+; RV32IM-NEXT: srli a1, a1, 8
+; RV32IM-NEXT: sb a2, 10(a0)
+; RV32IM-NEXT: sb a1, 11(a0)
+; RV32IM-NEXT: srli a1, s2, 8
+; RV32IM-NEXT: srli a2, s3, 8
+; RV32IM-NEXT: sb a1, 4(a0)
+; RV32IM-NEXT: srli a1, t0, 8
+; RV32IM-NEXT: sb a2, 5(a0)
+; RV32IM-NEXT: srli s0, s0, 8
+; RV32IM-NEXT: sb a1, 6(a0)
+; RV32IM-NEXT: sb s0, 7(a0)
+; RV32IM-NEXT: srli a3, a3, 8
+; RV32IM-NEXT: srli a5, a5, 8
+; RV32IM-NEXT: sb a3, 0(a0)
+; RV32IM-NEXT: srli a1, a6, 8
+; RV32IM-NEXT: sb a5, 1(a0)
+; RV32IM-NEXT: srli a2, a7, 8
+; RV32IM-NEXT: sb a1, 2(a0)
+; RV32IM-NEXT: sb a2, 3(a0)
+; RV32IM-NEXT: lw s0, 44(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s2, 36(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s5, 24(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s7, 16(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s8, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s9, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s10, 4(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s11, 0(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 48
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v16i8:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -96
+; RV64IM-NEXT: sd s0, 88(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 80(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s2, 72(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s3, 64(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s4, 56(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s5, 48(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s6, 40(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s7, 32(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s8, 24(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s9, 16(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s10, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s11, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lbu t3, 96(a2)
+; RV64IM-NEXT: lbu t2, 104(a2)
+; RV64IM-NEXT: lbu t1, 112(a2)
+; RV64IM-NEXT: lbu a4, 120(a2)
+; RV64IM-NEXT: lbu t6, 96(a1)
+; RV64IM-NEXT: lbu t5, 104(a1)
+; RV64IM-NEXT: lbu t4, 112(a1)
+; RV64IM-NEXT: lbu t0, 120(a1)
+; RV64IM-NEXT: lbu s2, 32(a2)
+; RV64IM-NEXT: lbu s3, 40(a2)
+; RV64IM-NEXT: lbu s1, 48(a2)
+; RV64IM-NEXT: lbu s0, 56(a2)
+; RV64IM-NEXT: lbu a3, 0(a2)
+; RV64IM-NEXT: lbu a5, 0(a1)
+; RV64IM-NEXT: lbu a6, 8(a1)
+; RV64IM-NEXT: lbu a7, 8(a2)
+; RV64IM-NEXT: lbu s4, 16(a2)
+; RV64IM-NEXT: lbu s5, 24(a2)
+; RV64IM-NEXT: lbu s6, 16(a1)
+; RV64IM-NEXT: lbu s7, 24(a1)
+; RV64IM-NEXT: lbu s8, 32(a1)
+; RV64IM-NEXT: mul a3, a5, a3
+; RV64IM-NEXT: mul a5, a6, a7
+; RV64IM-NEXT: lbu s9, 40(a1)
+; RV64IM-NEXT: lbu s10, 48(a1)
+; RV64IM-NEXT: lbu s11, 56(a1)
+; RV64IM-NEXT: mul a6, s6, s4
+; RV64IM-NEXT: mul a7, s7, s5
+; RV64IM-NEXT: mul s2, s8, s2
+; RV64IM-NEXT: lbu s4, 64(a2)
+; RV64IM-NEXT: lbu s5, 72(a2)
+; RV64IM-NEXT: lbu s6, 80(a2)
+; RV64IM-NEXT: lbu a2, 88(a2)
+; RV64IM-NEXT: mul t3, t6, t3
+; RV64IM-NEXT: mul t2, t5, t2
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: lbu t4, 64(a1)
+; RV64IM-NEXT: lbu t5, 72(a1)
+; RV64IM-NEXT: lbu t6, 80(a1)
+; RV64IM-NEXT: lbu a1, 88(a1)
+; RV64IM-NEXT: mul s3, s9, s3
+; RV64IM-NEXT: mul a4, t0, a4
+; RV64IM-NEXT: mul t0, s10, s1
+; RV64IM-NEXT: mul s0, s11, s0
+; RV64IM-NEXT: mul t4, t4, s4
+; RV64IM-NEXT: mul t5, t5, s5
+; RV64IM-NEXT: mul t6, t6, s6
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a2, t3, 8
+; RV64IM-NEXT: srli t2, t2, 8
+; RV64IM-NEXT: sb a2, 12(a0)
+; RV64IM-NEXT: srli a2, t1, 8
+; RV64IM-NEXT: sb t2, 13(a0)
+; RV64IM-NEXT: srli a4, a4, 8
+; RV64IM-NEXT: sb a2, 14(a0)
+; RV64IM-NEXT: sb a4, 15(a0)
+; RV64IM-NEXT: srli a2, t4, 8
+; RV64IM-NEXT: srli a4, t5, 8
+; RV64IM-NEXT: sb a2, 8(a0)
+; RV64IM-NEXT: srli a2, t6, 8
+; RV64IM-NEXT: sb a4, 9(a0)
+; RV64IM-NEXT: srli a1, a1, 8
+; RV64IM-NEXT: sb a2, 10(a0)
+; RV64IM-NEXT: sb a1, 11(a0)
+; RV64IM-NEXT: srli a1, s2, 8
+; RV64IM-NEXT: srli a2, s3, 8
+; RV64IM-NEXT: sb a1, 4(a0)
+; RV64IM-NEXT: srli a1, t0, 8
+; RV64IM-NEXT: sb a2, 5(a0)
+; RV64IM-NEXT: srli s0, s0, 8
+; RV64IM-NEXT: sb a1, 6(a0)
+; RV64IM-NEXT: sb s0, 7(a0)
+; RV64IM-NEXT: srli a3, a3, 8
+; RV64IM-NEXT: srli a5, a5, 8
+; RV64IM-NEXT: sb a3, 0(a0)
+; RV64IM-NEXT: srli a1, a6, 8
+; RV64IM-NEXT: sb a5, 1(a0)
+; RV64IM-NEXT: srli a2, a7, 8
+; RV64IM-NEXT: sb a1, 2(a0)
+; RV64IM-NEXT: sb a2, 3(a0)
+; RV64IM-NEXT: ld s0, 88(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 80(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s2, 72(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s3, 64(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s4, 56(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s5, 48(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s6, 40(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s7, 32(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s8, 24(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s9, 16(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s10, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s11, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 96
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v16i8:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <16 x i8> @llvm.umulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @umulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; RV32I-LABEL: umulh_v8i16:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -80
+; RV32I-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu s6, 16(a1)
+; RV32I-NEXT: lhu s5, 20(a1)
+; RV32I-NEXT: lhu a3, 24(a1)
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu a3, 28(a1)
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu s7, 16(a2)
+; RV32I-NEXT: lhu s8, 20(a2)
+; RV32I-NEXT: lhu a3, 24(a2)
+; RV32I-NEXT: sw a3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu a3, 28(a2)
+; RV32I-NEXT: sw a3, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lhu a3, 0(a1)
+; RV32I-NEXT: lhu s9, 4(a1)
+; RV32I-NEXT: lhu s10, 8(a1)
+; RV32I-NEXT: lhu s11, 12(a1)
+; RV32I-NEXT: lhu a1, 0(a2)
+; RV32I-NEXT: lhu s1, 4(a2)
+; RV32I-NEXT: lhu s2, 8(a2)
+; RV32I-NEXT: lhu s3, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s4, a0, 16
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s1, a0, 16
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s2, a0, 16
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s3, a0, 16
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: mv a1, s7
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s6, a0, 16
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a1, s8
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s5, a0, 16
+; RV32I-NEXT: lw a0, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: srli s7, a0, 16
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: call __mulsi3
+; RV32I-NEXT: sh s6, 8(s0)
+; RV32I-NEXT: srli a0, a0, 16
+; RV32I-NEXT: sh s5, 10(s0)
+; RV32I-NEXT: sh s7, 12(s0)
+; RV32I-NEXT: sh a0, 14(s0)
+; RV32I-NEXT: sh s4, 0(s0)
+; RV32I-NEXT: sh s1, 2(s0)
+; RV32I-NEXT: sh s2, 4(s0)
+; RV32I-NEXT: sh s3, 6(s0)
+; RV32I-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 80
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v8i16:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -144
+; RV64I-NEXT: sd ra, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s10, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s11, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu s6, 32(a1)
+; RV64I-NEXT: lhu s5, 40(a1)
+; RV64I-NEXT: lhu a3, 48(a1)
+; RV64I-NEXT: sd a3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu a3, 56(a1)
+; RV64I-NEXT: sd a3, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu s7, 32(a2)
+; RV64I-NEXT: lhu s8, 40(a2)
+; RV64I-NEXT: lhu a3, 48(a2)
+; RV64I-NEXT: sd a3, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu a3, 56(a2)
+; RV64I-NEXT: sd a3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lhu a3, 0(a1)
+; RV64I-NEXT: lhu s9, 8(a1)
+; RV64I-NEXT: lhu s10, 16(a1)
+; RV64I-NEXT: lhu s11, 24(a1)
+; RV64I-NEXT: lhu a1, 0(a2)
+; RV64I-NEXT: lhu s1, 8(a2)
+; RV64I-NEXT: lhu s2, 16(a2)
+; RV64I-NEXT: lhu s3, 24(a2)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s4, a0, 16
+; RV64I-NEXT: mv a0, s9
+; RV64I-NEXT: mv a1, s1
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 16
+; RV64I-NEXT: mv a0, s10
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s2, a0, 16
+; RV64I-NEXT: mv a0, s11
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s3, a0, 16
+; RV64I-NEXT: mv a0, s6
+; RV64I-NEXT: mv a1, s7
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s6, a0, 16
+; RV64I-NEXT: mv a0, s5
+; RV64I-NEXT: mv a1, s8
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 16
+; RV64I-NEXT: ld a0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 16
+; RV64I-NEXT: ld a0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld a1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sh s6, 8(s0)
+; RV64I-NEXT: srli a0, a0, 16
+; RV64I-NEXT: sh s5, 10(s0)
+; RV64I-NEXT: sh s7, 12(s0)
+; RV64I-NEXT: sh a0, 14(s0)
+; RV64I-NEXT: sh s4, 0(s0)
+; RV64I-NEXT: sh s1, 2(s0)
+; RV64I-NEXT: sh s2, 4(s0)
+; RV64I-NEXT: sh s3, 6(s0)
+; RV64I-NEXT: ld ra, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s10, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s11, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 144
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v8i16:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: addi sp, sp, -16
+; RV32IM-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: sw s1, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT: lhu a3, 16(a2)
+; RV32IM-NEXT: lhu a4, 20(a2)
+; RV32IM-NEXT: lhu a5, 24(a2)
+; RV32IM-NEXT: lhu a6, 28(a2)
+; RV32IM-NEXT: lhu a7, 0(a2)
+; RV32IM-NEXT: lhu t0, 4(a2)
+; RV32IM-NEXT: lhu t1, 8(a2)
+; RV32IM-NEXT: lhu a2, 12(a2)
+; RV32IM-NEXT: lhu t2, 0(a1)
+; RV32IM-NEXT: lhu t3, 4(a1)
+; RV32IM-NEXT: lhu t4, 8(a1)
+; RV32IM-NEXT: lhu t5, 12(a1)
+; RV32IM-NEXT: lhu t6, 16(a1)
+; RV32IM-NEXT: lhu s0, 20(a1)
+; RV32IM-NEXT: lhu s1, 24(a1)
+; RV32IM-NEXT: lhu a1, 28(a1)
+; RV32IM-NEXT: mul a7, t2, a7
+; RV32IM-NEXT: mul t0, t3, t0
+; RV32IM-NEXT: mul t1, t4, t1
+; RV32IM-NEXT: mul a2, t5, a2
+; RV32IM-NEXT: mul a3, t6, a3
+; RV32IM-NEXT: mul a4, s0, a4
+; RV32IM-NEXT: mul a5, s1, a5
+; RV32IM-NEXT: mul a1, a1, a6
+; RV32IM-NEXT: srli a6, a7, 16
+; RV32IM-NEXT: srli a7, t0, 16
+; RV32IM-NEXT: srli t0, t1, 16
+; RV32IM-NEXT: srli a2, a2, 16
+; RV32IM-NEXT: srli a3, a3, 16
+; RV32IM-NEXT: srli a4, a4, 16
+; RV32IM-NEXT: srli a5, a5, 16
+; RV32IM-NEXT: srli a1, a1, 16
+; RV32IM-NEXT: sh a3, 8(a0)
+; RV32IM-NEXT: sh a4, 10(a0)
+; RV32IM-NEXT: sh a5, 12(a0)
+; RV32IM-NEXT: sh a1, 14(a0)
+; RV32IM-NEXT: sh a6, 0(a0)
+; RV32IM-NEXT: sh a7, 2(a0)
+; RV32IM-NEXT: sh t0, 4(a0)
+; RV32IM-NEXT: sh a2, 6(a0)
+; RV32IM-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: lw s1, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT: addi sp, sp, 16
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v8i16:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: addi sp, sp, -16
+; RV64IM-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: sd s1, 0(sp) # 8-byte Folded Spill
+; RV64IM-NEXT: lhu a3, 32(a2)
+; RV64IM-NEXT: lhu a4, 40(a2)
+; RV64IM-NEXT: lhu a5, 48(a2)
+; RV64IM-NEXT: lhu a6, 56(a2)
+; RV64IM-NEXT: lhu a7, 0(a2)
+; RV64IM-NEXT: lhu t0, 8(a2)
+; RV64IM-NEXT: lhu t1, 16(a2)
+; RV64IM-NEXT: lhu a2, 24(a2)
+; RV64IM-NEXT: lhu t2, 0(a1)
+; RV64IM-NEXT: lhu t3, 8(a1)
+; RV64IM-NEXT: lhu t4, 16(a1)
+; RV64IM-NEXT: lhu t5, 24(a1)
+; RV64IM-NEXT: lhu t6, 32(a1)
+; RV64IM-NEXT: lhu s0, 40(a1)
+; RV64IM-NEXT: lhu s1, 48(a1)
+; RV64IM-NEXT: lhu a1, 56(a1)
+; RV64IM-NEXT: mul a7, t2, a7
+; RV64IM-NEXT: mul t0, t3, t0
+; RV64IM-NEXT: mul t1, t4, t1
+; RV64IM-NEXT: mul a2, t5, a2
+; RV64IM-NEXT: mul a3, t6, a3
+; RV64IM-NEXT: mul a4, s0, a4
+; RV64IM-NEXT: mul a5, s1, a5
+; RV64IM-NEXT: mul a1, a1, a6
+; RV64IM-NEXT: srli a6, a7, 16
+; RV64IM-NEXT: srli a7, t0, 16
+; RV64IM-NEXT: srli t0, t1, 16
+; RV64IM-NEXT: srli a2, a2, 16
+; RV64IM-NEXT: srli a3, a3, 16
+; RV64IM-NEXT: srli a4, a4, 16
+; RV64IM-NEXT: srli a5, a5, 16
+; RV64IM-NEXT: srli a1, a1, 16
+; RV64IM-NEXT: sh a3, 8(a0)
+; RV64IM-NEXT: sh a4, 10(a0)
+; RV64IM-NEXT: sh a5, 12(a0)
+; RV64IM-NEXT: sh a1, 14(a0)
+; RV64IM-NEXT: sh a6, 0(a0)
+; RV64IM-NEXT: sh a7, 2(a0)
+; RV64IM-NEXT: sh t0, 4(a0)
+; RV64IM-NEXT: sh a2, 6(a0)
+; RV64IM-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: ld s1, 0(sp) # 8-byte Folded Reload
+; RV64IM-NEXT: addi sp, sp, 16
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v8i16:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <8 x i16> @llvm.umulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @umulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; RV32I-LABEL: umulh_v4i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 0(a1)
+; RV32I-NEXT: lw s0, 4(a1)
+; RV32I-NEXT: lw s1, 8(a1)
+; RV32I-NEXT: lw s2, 12(a1)
+; RV32I-NEXT: lw a4, 0(a2)
+; RV32I-NEXT: lw s3, 4(a2)
+; RV32I-NEXT: lw s4, 8(a2)
+; RV32I-NEXT: lw s5, 12(a2)
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, a4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw s7, 0(s6)
+; RV32I-NEXT: sw s0, 4(s6)
+; RV32I-NEXT: sw s1, 8(s6)
+; RV32I-NEXT: sw a1, 12(s6)
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v4i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -80
+; RV64I-NEXT: sd ra, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lwu a3, 0(a1)
+; RV64I-NEXT: lwu s0, 8(a1)
+; RV64I-NEXT: lwu s1, 16(a1)
+; RV64I-NEXT: lwu s2, 24(a1)
+; RV64I-NEXT: lwu a1, 0(a2)
+; RV64I-NEXT: lwu s3, 8(a2)
+; RV64I-NEXT: lwu s4, 16(a2)
+; RV64I-NEXT: lwu s5, 24(a2)
+; RV64I-NEXT: mv s6, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s7, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s0, a0, 32
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s4
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s1, a0, 32
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: mv a1, s5
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: sw s7, 0(s6)
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: sw s0, 4(s6)
+; RV64I-NEXT: sw s1, 8(s6)
+; RV64I-NEXT: sw a0, 12(s6)
+; RV64I-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 80
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v4i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a2)
+; RV32IM-NEXT: lw a4, 4(a2)
+; RV32IM-NEXT: lw a5, 8(a2)
+; RV32IM-NEXT: lw a2, 12(a2)
+; RV32IM-NEXT: lw a6, 0(a1)
+; RV32IM-NEXT: lw a7, 4(a1)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: mulhu a3, a6, a3
+; RV32IM-NEXT: mulhu a4, a7, a4
+; RV32IM-NEXT: mulhu a5, t0, a5
+; RV32IM-NEXT: mulhu a1, a1, a2
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a4, 4(a0)
+; RV32IM-NEXT: sw a5, 8(a0)
+; RV32IM-NEXT: sw a1, 12(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v4i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lwu a3, 0(a2)
+; RV64IM-NEXT: lwu a4, 8(a2)
+; RV64IM-NEXT: lwu a5, 16(a2)
+; RV64IM-NEXT: lwu a2, 24(a2)
+; RV64IM-NEXT: lwu a6, 0(a1)
+; RV64IM-NEXT: lwu a7, 8(a1)
+; RV64IM-NEXT: lwu t0, 16(a1)
+; RV64IM-NEXT: lwu a1, 24(a1)
+; RV64IM-NEXT: mul a3, a6, a3
+; RV64IM-NEXT: mul a4, a7, a4
+; RV64IM-NEXT: mul a5, t0, a5
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 32
+; RV64IM-NEXT: srli a4, a4, 32
+; RV64IM-NEXT: srli a5, a5, 32
+; RV64IM-NEXT: srli a1, a1, 32
+; RV64IM-NEXT: sw a3, 0(a0)
+; RV64IM-NEXT: sw a4, 4(a0)
+; RV64IM-NEXT: sw a5, 8(a0)
+; RV64IM-NEXT: sw a1, 12(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v4i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <4 x i32> @llvm.umulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @umulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; RV32I-LABEL: umulh_v2i64:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -48
+; RV32I-NEXT: sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s5, 0(a2)
+; RV32I-NEXT: lw s6, 4(a2)
+; RV32I-NEXT: lw s4, 8(a2)
+; RV32I-NEXT: lw s1, 12(a2)
+; RV32I-NEXT: lw s7, 0(a1)
+; RV32I-NEXT: lw s8, 4(a1)
+; RV32I-NEXT: lw s3, 8(a1)
+; RV32I-NEXT: lw s2, 12(a1)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s9, a1
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: sltu a0, s9, a0
+; RV32I-NEXT: add s5, a1, a0
+; RV32I-NEXT: mv a0, s7
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s9, a0, s9
+; RV32I-NEXT: sltu a0, s9, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s7, s5, a0
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, a0, s7
+; RV32I-NEXT: sltu a2, s7, s5
+; RV32I-NEXT: sltu a0, s6, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s7, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add s4, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s3, s4, a0
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s3, s4
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw s6, 0(s0)
+; RV32I-NEXT: sw s7, 4(s0)
+; RV32I-NEXT: sw s3, 8(s0)
+; RV32I-NEXT: sw a0, 12(s0)
+; RV32I-NEXT: lw ra, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 48
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v2i64:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a3
+; RV64I-NEXT: mv s1, a1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v2i64:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a1)
+; RV32IM-NEXT: lw a4, 4(a1)
+; RV32IM-NEXT: lw a5, 0(a2)
+; RV32IM-NEXT: lw a6, 4(a2)
+; RV32IM-NEXT: lw a7, 8(a2)
+; RV32IM-NEXT: lw a2, 12(a2)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: mulhu t1, a3, a5
+; RV32IM-NEXT: mul t2, a4, a5
+; RV32IM-NEXT: mulhu a5, a4, a5
+; RV32IM-NEXT: mul t3, a3, a6
+; RV32IM-NEXT: mulhu a3, a3, a6
+; RV32IM-NEXT: mul t4, a4, a6
+; RV32IM-NEXT: mulhu a4, a4, a6
+; RV32IM-NEXT: add t1, t2, t1
+; RV32IM-NEXT: mulhu a6, t0, a7
+; RV32IM-NEXT: sltu t2, t1, t2
+; RV32IM-NEXT: mul t5, a1, a7
+; RV32IM-NEXT: add a5, a5, t2
+; RV32IM-NEXT: mul t2, t0, a2
+; RV32IM-NEXT: add t1, t3, t1
+; RV32IM-NEXT: sltu t1, t1, t3
+; RV32IM-NEXT: mulhu a7, a1, a7
+; RV32IM-NEXT: mulhu t0, t0, a2
+; RV32IM-NEXT: add a3, a3, t1
+; RV32IM-NEXT: mul t1, a1, a2
+; RV32IM-NEXT: add a3, a5, a3
+; RV32IM-NEXT: mulhu a1, a1, a2
+; RV32IM-NEXT: sltu a2, a3, a5
+; RV32IM-NEXT: add a3, t4, a3
+; RV32IM-NEXT: add a6, t5, a6
+; RV32IM-NEXT: sltu a5, a6, t5
+; RV32IM-NEXT: add a6, t2, a6
+; RV32IM-NEXT: sltu a6, a6, t2
+; RV32IM-NEXT: sltu t2, a3, t4
+; RV32IM-NEXT: add a5, a7, a5
+; RV32IM-NEXT: add a6, t0, a6
+; RV32IM-NEXT: add a2, a4, a2
+; RV32IM-NEXT: add a6, a5, a6
+; RV32IM-NEXT: add a4, t1, a6
+; RV32IM-NEXT: sltu a5, a6, a5
+; RV32IM-NEXT: sltu a6, a4, t1
+; RV32IM-NEXT: add a1, a1, a5
+; RV32IM-NEXT: add a2, a2, t2
+; RV32IM-NEXT: add a1, a1, a6
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a2, 4(a0)
+; RV32IM-NEXT: sw a4, 8(a0)
+; RV32IM-NEXT: sw a1, 12(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v2i64:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: mulhu a0, a0, a2
+; RV64IM-NEXT: mulhu a1, a1, a3
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v2i64:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <2 x i64> @llvm.umulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @umulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; RV32I-LABEL: umulh_v3i32:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -32
+; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 0(a1)
+; RV32I-NEXT: lw s0, 4(a1)
+; RV32I-NEXT: lw s1, 8(a1)
+; RV32I-NEXT: lw a4, 0(a2)
+; RV32I-NEXT: lw s2, 4(a2)
+; RV32I-NEXT: lw s3, 8(a2)
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: mv a0, a3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, a4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sw s5, 0(s4)
+; RV32I-NEXT: sw s0, 4(s4)
+; RV32I-NEXT: sw a1, 8(s4)
+; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 32
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v3i32:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -64
+; RV64I-NEXT: sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: lwu a3, 0(a1)
+; RV64I-NEXT: lwu s0, 8(a1)
+; RV64I-NEXT: lwu s1, 16(a1)
+; RV64I-NEXT: lwu a1, 0(a2)
+; RV64I-NEXT: lwu s2, 8(a2)
+; RV64I-NEXT: lwu s3, 16(a2)
+; RV64I-NEXT: mv s4, a0
+; RV64I-NEXT: mv a0, a3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli s5, a0, 32
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: mv a1, s2
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: slli a0, a0, 32
+; RV64I-NEXT: or s0, s5, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: mv a1, s3
+; RV64I-NEXT: call __muldi3
+; RV64I-NEXT: srli a0, a0, 32
+; RV64I-NEXT: sd s0, 0(s4)
+; RV64I-NEXT: sw a0, 8(s4)
+; RV64I-NEXT: ld ra, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 64
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v3i32:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a2)
+; RV32IM-NEXT: lw a4, 4(a2)
+; RV32IM-NEXT: lw a2, 8(a2)
+; RV32IM-NEXT: lw a5, 0(a1)
+; RV32IM-NEXT: lw a6, 4(a1)
+; RV32IM-NEXT: lw a1, 8(a1)
+; RV32IM-NEXT: mulhu a3, a5, a3
+; RV32IM-NEXT: mulhu a4, a6, a4
+; RV32IM-NEXT: mulhu a1, a1, a2
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sw a4, 4(a0)
+; RV32IM-NEXT: sw a1, 8(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v3i32:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: lwu a3, 8(a2)
+; RV64IM-NEXT: lwu a4, 8(a1)
+; RV64IM-NEXT: lwu a5, 0(a1)
+; RV64IM-NEXT: lwu a6, 0(a2)
+; RV64IM-NEXT: lwu a2, 16(a2)
+; RV64IM-NEXT: lwu a1, 16(a1)
+; RV64IM-NEXT: mul a3, a4, a3
+; RV64IM-NEXT: mul a4, a5, a6
+; RV64IM-NEXT: mul a1, a1, a2
+; RV64IM-NEXT: srli a3, a3, 32
+; RV64IM-NEXT: srli a4, a4, 32
+; RV64IM-NEXT: slli a3, a3, 32
+; RV64IM-NEXT: or a3, a4, a3
+; RV64IM-NEXT: srli a1, a1, 32
+; RV64IM-NEXT: sd a3, 0(a0)
+; RV64IM-NEXT: sw a1, 8(a0)
+; RV64IM-NEXT: ret
+;
+; CHECK-V-LABEL: umulh_v3i32:
+; CHECK-V: # %bb.0:
+; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-V-NEXT: vmulhu.vv v8, v8, v9
+; CHECK-V-NEXT: ret
+ %r = call <3 x i32> @llvm.umulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @umulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; RV32I-LABEL: umulh_v2i48:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -64
+; RV32I-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s8, 0(a1)
+; RV32I-NEXT: lw s11, 4(a1)
+; RV32I-NEXT: lw s10, 8(a1)
+; RV32I-NEXT: lw s9, 12(a1)
+; RV32I-NEXT: lw s3, 0(a2)
+; RV32I-NEXT: lhu s4, 4(a2)
+; RV32I-NEXT: lw s2, 8(a2)
+; RV32I-NEXT: lhu s1, 12(a2)
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: slli s5, s8, 16
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s7, a1
+; RV32I-NEXT: mv a0, s5
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: srli a0, s8, 16
+; RV32I-NEXT: slli s11, s11, 16
+; RV32I-NEXT: or s8, s11, a0
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s5, a0, s5
+; RV32I-NEXT: add a2, s6, s5
+; RV32I-NEXT: sltu a2, a2, s6
+; RV32I-NEXT: sltu a0, s5, a0
+; RV32I-NEXT: add a2, s7, a2
+; RV32I-NEXT: add s3, a1, a0
+; RV32I-NEXT: add s5, s3, a2
+; RV32I-NEXT: mv a0, s8
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, a0, s5
+; RV32I-NEXT: sltu a2, s5, s3
+; RV32I-NEXT: sltu s8, s7, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s8, a1, s8
+; RV32I-NEXT: slli s3, s10, 16
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: srli a0, s10, 16
+; RV32I-NEXT: slli s9, s9, 16
+; RV32I-NEXT: or s6, s9, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s2
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a2, s4, s3
+; RV32I-NEXT: sltu a2, a2, s4
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: add a2, s5, a2
+; RV32I-NEXT: add s2, a1, a0
+; RV32I-NEXT: add s3, s2, a2
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s3, s2
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: sltu a0, s3, a0
+; RV32I-NEXT: srli a2, s3, 16
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw s7, 0(s0)
+; RV32I-NEXT: sh s8, 4(s0)
+; RV32I-NEXT: sh s3, 6(s0)
+; RV32I-NEXT: sh a2, 8(s0)
+; RV32I-NEXT: sh a0, 10(s0)
+; RV32I-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 64
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v2i48:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -32
+; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT: mv s0, a1
+; RV64I-NEXT: li a1, -1
+; RV64I-NEXT: srli a1, a1, 16
+; RV64I-NEXT: and s1, a3, a1
+; RV64I-NEXT: and a2, a2, a1
+; RV64I-NEXT: slli a0, a0, 16
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s2, a1
+; RV64I-NEXT: slli a0, s0, 16
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 32
+; RV64I-NEXT: ret
+;
+; RV32IM-LABEL: umulh_v2i48:
+; RV32IM: # %bb.0:
+; RV32IM-NEXT: lw a3, 0(a1)
+; RV32IM-NEXT: lw a4, 4(a1)
+; RV32IM-NEXT: lw a5, 0(a2)
+; RV32IM-NEXT: lhu a6, 4(a2)
+; RV32IM-NEXT: lw a7, 8(a2)
+; RV32IM-NEXT: lhu a2, 12(a2)
+; RV32IM-NEXT: lw t0, 8(a1)
+; RV32IM-NEXT: lw a1, 12(a1)
+; RV32IM-NEXT: srli t1, a3, 16
+; RV32IM-NEXT: slli a4, a4, 16
+; RV32IM-NEXT: slli a3, a3, 16
+; RV32IM-NEXT: or a4, a4, t1
+; RV32IM-NEXT: mulhu t1, a3, a5
+; RV32IM-NEXT: mul t2, a4, a5
+; RV32IM-NEXT: mulhu a5, a4, a5
+; RV32IM-NEXT: mul t3, a3, a6
+; RV32IM-NEXT: slli t4, t0, 16
+; RV32IM-NEXT: srli t0, t0, 16
+; RV32IM-NEXT: slli a1, a1, 16
+; RV32IM-NEXT: mulhu a3, a3, a6
+; RV32IM-NEXT: mulhu t5, t4, a7
+; RV32IM-NEXT: add t1, t2, t1
+; RV32IM-NEXT: or a1, a1, t0
+; RV32IM-NEXT: sltu t0, t1, t2
+; RV32IM-NEXT: mul t2, a4, a6
+; RV32IM-NEXT: add a5, a5, t0
+; RV32IM-NEXT: mul t0, a1, a7
+; RV32IM-NEXT: add t1, t3, t1
+; RV32IM-NEXT: sltu t1, t1, t3
+; RV32IM-NEXT: mul t3, t4, a2
+; RV32IM-NEXT: add a3, a3, t1
+; RV32IM-NEXT: mulhu a7, a1, a7
+; RV32IM-NEXT: add a3, a5, a3
+; RV32IM-NEXT: mulhu t1, t4, a2
+; RV32IM-NEXT: mulhu a4, a4, a6
+; RV32IM-NEXT: mul a6, a1, a2
+; RV32IM-NEXT: mulhu a1, a1, a2
+; RV32IM-NEXT: sltu a2, a3, a5
+; RV32IM-NEXT: add a3, t2, a3
+; RV32IM-NEXT: add t5, t0, t5
+; RV32IM-NEXT: sltu a5, t5, t0
+; RV32IM-NEXT: add t5, t3, t5
+; RV32IM-NEXT: sltu t0, t5, t3
+; RV32IM-NEXT: sltu t2, a3, t2
+; RV32IM-NEXT: add a5, a7, a5
+; RV32IM-NEXT: add t0, t1, t0
+; RV32IM-NEXT: add a2, a4, a2
+; RV32IM-NEXT: add t0, a5, t0
+; RV32IM-NEXT: add a4, a6, t0
+; RV32IM-NEXT: sltu a5, t0, a5
+; RV32IM-NEXT: sltu a6, a4, a6
+; RV32IM-NEXT: add a1, a1, a5
+; RV32IM-NEXT: add a2, a2, t2
+; RV32IM-NEXT: add a1, a1, a6
+; RV32IM-NEXT: srli a5, a4, 16
+; RV32IM-NEXT: sw a3, 0(a0)
+; RV32IM-NEXT: sh a2, 4(a0)
+; RV32IM-NEXT: sh a4, 6(a0)
+; RV32IM-NEXT: sh a5, 8(a0)
+; RV32IM-NEXT: sh a1, 10(a0)
+; RV32IM-NEXT: ret
+;
+; RV64IM-LABEL: umulh_v2i48:
+; RV64IM: # %bb.0:
+; RV64IM-NEXT: li a4, -1
+; RV64IM-NEXT: srli a4, a4, 16
+; RV64IM-NEXT: and a3, a3, a4
+; RV64IM-NEXT: and a2, a2, a4
+; RV64IM-NEXT: slli a0, a0, 16
+; RV64IM-NEXT: slli a1, a1, 16
+; RV64IM-NEXT: mulhu a0, a0, a2
+; RV64IM-NEXT: mulhu a1, a1, a3
+; RV64IM-NEXT: ret
+;
+; RV32V-LABEL: umulh_v2i48:
+; RV32V: # %bb.0:
+; RV32V-NEXT: addi sp, sp, -16
+; RV32V-NEXT: li a0, -1
+; RV32V-NEXT: lui a1, 16
+; RV32V-NEXT: addi a1, a1, -1
+; RV32V-NEXT: sw a0, 8(sp)
+; RV32V-NEXT: sw a1, 12(sp)
+; RV32V-NEXT: addi a0, sp, 8
+; RV32V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32V-NEXT: vlse64.v v10, (a0), zero
+; RV32V-NEXT: vand.vv v9, v9, v10
+; RV32V-NEXT: vsll.vi v8, v8, 16
+; RV32V-NEXT: vmulhu.vv v8, v8, v9
+; RV32V-NEXT: addi sp, sp, 16
+; RV32V-NEXT: ret
+;
+; RV64V-LABEL: umulh_v2i48:
+; RV64V: # %bb.0:
+; RV64V-NEXT: li a0, -1
+; RV64V-NEXT: srli a0, a0, 16
+; RV64V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64V-NEXT: vand.vx v9, v9, a0
+; RV64V-NEXT: vsll.vi v8, v8, 16
+; RV64V-NEXT: vmulhu.vv v8, v8, v9
+; RV64V-NEXT: ret
+ %r = call <2 x i48> @llvm.umulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @umulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; RV32I-LABEL: umulh_v2i128:
+; RV32I: # %bb.0:
+; RV32I-NEXT: addi sp, sp, -128
+; RV32I-NEXT: sw ra, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s0, 120(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s1, 116(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s2, 112(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s3, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 104(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s5, 100(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s6, 96(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s7, 92(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s8, 88(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s9, 84(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s10, 80(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s11, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 16(a2)
+; RV32I-NEXT: sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 20(a2)
+; RV32I-NEXT: sw a3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 24(a2)
+; RV32I-NEXT: sw a3, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 28(a2)
+; RV32I-NEXT: sw a3, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 16(a1)
+; RV32I-NEXT: sw a3, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 20(a1)
+; RV32I-NEXT: sw a3, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 24(a1)
+; RV32I-NEXT: sw a3, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a3, 28(a1)
+; RV32I-NEXT: sw a3, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s5, 0(a2)
+; RV32I-NEXT: lw s4, 4(a2)
+; RV32I-NEXT: lw a3, 8(a2)
+; RV32I-NEXT: sw a3, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw a2, 12(a2)
+; RV32I-NEXT: sw a2, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: lw s3, 0(a1)
+; RV32I-NEXT: lw s2, 4(a1)
+; RV32I-NEXT: lw s9, 8(a1)
+; RV32I-NEXT: lw s11, 12(a1)
+; RV32I-NEXT: sw a0, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, a0, s6
+; RV32I-NEXT: sltu a0, s6, a0
+; RV32I-NEXT: add s0, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, a0, s6
+; RV32I-NEXT: sltu a0, s6, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, s0, a0
+; RV32I-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s2, a0, s1
+; RV32I-NEXT: sltu a2, s1, s0
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s0, a1, a0
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s8, a1
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s6, a0
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: add s8, a0, s8
+; RV32I-NEXT: sw s9, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s1, s7, s2
+; RV32I-NEXT: add a2, a0, s8
+; RV32I-NEXT: add s0, a2, s0
+; RV32I-NEXT: sltu s9, s1, s7
+; RV32I-NEXT: add s2, s0, s9
+; RV32I-NEXT: beq s2, a2, .LBB15_2
+; RV32I-NEXT: # %bb.1:
+; RV32I-NEXT: sltu s9, s2, a2
+; RV32I-NEXT: .LBB15_2:
+; RV32I-NEXT: sltu a3, s8, s6
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a3, s5, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s10, a3, a0
+; RV32I-NEXT: sw s11, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s4, a0
+; RV32I-NEXT: sw a1, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s5, a0, s10
+; RV32I-NEXT: add s9, s5, s9
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s6, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s8, a0
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s6
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s11, a0
+; RV32I-NEXT: mv s6, a1
+; RV32I-NEXT: add s7, a0, s0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s1, s8, s1
+; RV32I-NEXT: add a2, a0, s7
+; RV32I-NEXT: add a3, a2, s2
+; RV32I-NEXT: sltu s0, s1, s8
+; RV32I-NEXT: add a3, a3, s0
+; RV32I-NEXT: beq a3, a2, .LBB15_4
+; RV32I-NEXT: # %bb.3:
+; RV32I-NEXT: sltu s0, a3, a2
+; RV32I-NEXT: .LBB15_4:
+; RV32I-NEXT: sltu a3, s7, s11
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s6, s6, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s1, s6, a0
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s8, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s9, s5
+; RV32I-NEXT: add a3, a0, s1
+; RV32I-NEXT: sltu a4, s5, s4
+; RV32I-NEXT: add s0, a3, s0
+; RV32I-NEXT: lw a5, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sltu a5, s10, a5
+; RV32I-NEXT: add s7, s9, s0
+; RV32I-NEXT: lw a6, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a5, a6, a5
+; RV32I-NEXT: add a2, a4, a2
+; RV32I-NEXT: sltu a4, s1, s6
+; RV32I-NEXT: sltu a0, a3, a0
+; RV32I-NEXT: add a1, a1, a4
+; RV32I-NEXT: sltu a3, s0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s4, a5, a2
+; RV32I-NEXT: add a0, a0, a3
+; RV32I-NEXT: sltu s9, s7, s9
+; RV32I-NEXT: add a0, s4, a0
+; RV32I-NEXT: add s5, a0, s9
+; RV32I-NEXT: lw s10, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: mv a0, s10
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s8
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s7, s0, s7
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: sw s7, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu s8, s7, s0
+; RV32I-NEXT: add a3, a2, s8
+; RV32I-NEXT: add a3, a3, s5
+; RV32I-NEXT: beq a3, a2, .LBB15_6
+; RV32I-NEXT: # %bb.5:
+; RV32I-NEXT: sltu s8, a3, a2
+; RV32I-NEXT: .LBB15_6:
+; RV32I-NEXT: sw a3, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT: beq s5, s4, .LBB15_8
+; RV32I-NEXT: # %bb.7:
+; RV32I-NEXT: sltu s9, s5, s4
+; RV32I-NEXT: .LBB15_8:
+; RV32I-NEXT: sltu a3, s3, s2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s1, s1, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s1, a0
+; RV32I-NEXT: mv a0, s6
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s10, a0, s0
+; RV32I-NEXT: sltu s5, s0, s1
+; RV32I-NEXT: add s9, s10, s9
+; RV32I-NEXT: sw s9, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a0, s10, a0
+; RV32I-NEXT: sw a0, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s5, a1, s5
+; RV32I-NEXT: lw s3, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a1
+; RV32I-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add s1, a1, a0
+; RV32I-NEXT: mv a0, s3
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s11, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s0, a0, s0
+; RV32I-NEXT: sltu a0, s0, a0
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s0, s1, a0
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, a0, s0
+; RV32I-NEXT: sltu a2, s0, s1
+; RV32I-NEXT: sltu a0, s6, a0
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s7, a1, a0
+; RV32I-NEXT: lw s9, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s4
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: add s3, a0, s3
+; RV32I-NEXT: mv a0, s9
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s11
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, s0, s6
+; RV32I-NEXT: add a2, a0, s3
+; RV32I-NEXT: add s7, a2, s7
+; RV32I-NEXT: sltu s11, s6, s0
+; RV32I-NEXT: add s9, s7, s11
+; RV32I-NEXT: beq s9, a2, .LBB15_10
+; RV32I-NEXT: # %bb.9:
+; RV32I-NEXT: sltu s11, s9, a2
+; RV32I-NEXT: .LBB15_10:
+; RV32I-NEXT: lw a3, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s8, a3, s8
+; RV32I-NEXT: sw s8, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a3, a3, s10
+; RV32I-NEXT: sw a3, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a3, s3, s2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add a3, s1, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a1, s5, a1
+; RV32I-NEXT: sw a1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw a3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s10, a3, a0
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s8, a0
+; RV32I-NEXT: sw a1, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT: add s2, a0, s10
+; RV32I-NEXT: add s11, s2, s11
+; RV32I-NEXT: lw s1, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s3, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s5, a1
+; RV32I-NEXT: lw a0, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s3
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s7, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: add s4, a0, s5
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s6, s0, s6
+; RV32I-NEXT: add a2, a0, s4
+; RV32I-NEXT: add a3, a2, s9
+; RV32I-NEXT: sltu s0, s6, s0
+; RV32I-NEXT: add a3, a3, s0
+; RV32I-NEXT: beq a3, a2, .LBB15_12
+; RV32I-NEXT: # %bb.11:
+; RV32I-NEXT: sltu s0, a3, a2
+; RV32I-NEXT: .LBB15_12:
+; RV32I-NEXT: lw a3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sltu a3, a3, a4
+; RV32I-NEXT: sw a3, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sltu a3, s4, s7
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s3, s3, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: lw s4, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw a1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s4, a1, s4
+; RV32I-NEXT: add s6, s3, a0
+; RV32I-NEXT: lw a0, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s5, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s11, s2
+; RV32I-NEXT: add a3, a0, s6
+; RV32I-NEXT: sltu a4, s2, s8
+; RV32I-NEXT: add s0, a3, s0
+; RV32I-NEXT: lw a5, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sltu a5, s10, a5
+; RV32I-NEXT: add s9, s11, s0
+; RV32I-NEXT: lw a6, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add a5, a6, a5
+; RV32I-NEXT: add a2, a4, a2
+; RV32I-NEXT: sltu a4, s6, s3
+; RV32I-NEXT: sltu a0, a3, a0
+; RV32I-NEXT: add a1, a1, a4
+; RV32I-NEXT: sltu a3, s0, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s7, a5, a2
+; RV32I-NEXT: add a0, a0, a3
+; RV32I-NEXT: sltu s6, s9, s11
+; RV32I-NEXT: add a0, s7, a0
+; RV32I-NEXT: add s8, a0, s6
+; RV32I-NEXT: lw s11, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: mv s2, a0
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: add s10, a0, s3
+; RV32I-NEXT: mv a0, s11
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: mv a2, s5
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: add s3, s0, s9
+; RV32I-NEXT: add a2, a0, s10
+; RV32I-NEXT: sltu s9, s3, s0
+; RV32I-NEXT: add s0, a2, s9
+; RV32I-NEXT: add s0, s0, s8
+; RV32I-NEXT: beq s0, a2, .LBB15_14
+; RV32I-NEXT: # %bb.13:
+; RV32I-NEXT: sltu s9, s0, a2
+; RV32I-NEXT: .LBB15_14:
+; RV32I-NEXT: lw a3, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT: add s4, s4, a3
+; RV32I-NEXT: beq s8, s7, .LBB15_16
+; RV32I-NEXT: # %bb.15:
+; RV32I-NEXT: sltu s6, s8, s7
+; RV32I-NEXT: .LBB15_16:
+; RV32I-NEXT: sltu a3, s10, s2
+; RV32I-NEXT: sltu a0, a2, a0
+; RV32I-NEXT: add s1, s1, a3
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: add s2, s1, a0
+; RV32I-NEXT: lw a0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a1, 0
+; RV32I-NEXT: lw a2, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT: li a3, 0
+; RV32I-NEXT: call __muldi3
+; RV32I-NEXT: sltu a2, s2, s1
+; RV32I-NEXT: add s2, a0, s2
+; RV32I-NEXT: add a1, a1, a2
+; RV32I-NEXT: add s6, s2, s6
+; RV32I-NEXT: sltu a0, s2, a0
+; RV32I-NEXT: add s9, s6, s9
+; RV32I-NEXT: sltu a2, s6, s2
+; RV32I-NEXT: add a0, a1, a0
+; RV32I-NEXT: sltu a1, s9, s6
+; RV32I-NEXT: add a0, a0, a2
+; RV32I-NEXT: add a0, a0, a1
+; RV32I-NEXT: lw a1, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw s3, 16(a1)
+; RV32I-NEXT: sw s0, 20(a1)
+; RV32I-NEXT: sw s9, 24(a1)
+; RV32I-NEXT: sw a0, 28(a1)
+; RV32I-NEXT: lw a0, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 0(a1)
+; RV32I-NEXT: lw a0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 4(a1)
+; RV32I-NEXT: lw a0, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT: sw a0, 8(a1)
+; RV32I-NEXT: sw s4, 12(a1)
+; RV32I-NEXT: lw ra, 124(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s0, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s1, 116(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s2, 112(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s3, 108(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s5, 100(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s6, 96(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s7, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s8, 88(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s9, 84(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s10, 80(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s11, 76(sp) # 4-byte Folded Reload
+; RV32I-NEXT: addi sp, sp, 128
+; RV32I-NEXT: ret
+;
+; RV64I-LABEL: umulh_v2i128:
+; RV64I: # %bb.0:
+; RV64I-NEXT: addi sp, sp, -96
+; RV64I-NEXT: sd ra, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s0, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s1, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s2, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s3, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s4, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s5, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s6, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s7, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s8, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT: sd s9, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT: ld s5, 0(a2)
+; RV64I-NEXT: ld s6, 8(a2)
+; RV64I-NEXT: ld s4, 16(a2)
+; RV64I-NEXT: ld s1, 24(a2)
+; RV64I-NEXT: ld s7, 0(a1)
+; RV64I-NEXT: ld s8, 8(a1)
+; RV64I-NEXT: ld s3, 16(a1)
+; RV64I-NEXT: ld s2, 24(a1)
+; RV64I-NEXT: mv s0, a0
+; RV64I-NEXT: mv a0, s7
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s5
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s9, a1
+; RV64I-NEXT: mv a0, s8
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s5
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s9, a0, s9
+; RV64I-NEXT: sltu a0, s9, a0
+; RV64I-NEXT: add s5, a1, a0
+; RV64I-NEXT: mv a0, s7
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s6
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s9, a0, s9
+; RV64I-NEXT: sltu a0, s9, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s7, s5, a0
+; RV64I-NEXT: mv a0, s8
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s6
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s6, a0, s7
+; RV64I-NEXT: sltu a2, s7, s5
+; RV64I-NEXT: sltu a0, s6, a0
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: add s7, a1, a0
+; RV64I-NEXT: mv a0, s3
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s4
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: mv s5, a1
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s4
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s5, a0, s5
+; RV64I-NEXT: sltu a0, s5, a0
+; RV64I-NEXT: add s4, a1, a0
+; RV64I-NEXT: mv a0, s3
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: add s5, a0, s5
+; RV64I-NEXT: sltu a0, s5, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: add s3, s4, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: li a1, 0
+; RV64I-NEXT: mv a2, s1
+; RV64I-NEXT: li a3, 0
+; RV64I-NEXT: call __multi3
+; RV64I-NEXT: sltu a2, s3, s4
+; RV64I-NEXT: add s3, a0, s3
+; RV64I-NEXT: add a1, a1, a2
+; RV64I-NEXT: sltu a0, s3, a0
+; RV64I-NEXT: add a0, a1, a0
+; RV64I-NEXT: sd s6, 0(s0)
+; RV64I-NEXT: sd s7, 8(s0)
+; RV64I-NEXT: sd s3, 16(s0)
+; RV64I-NEXT: sd a0, 24(s0)
+; RV64I-NEXT: ld ra, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s0, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s1, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s2, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s3, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s4, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s5, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s6, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s7, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s8, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT: ld s9, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT: addi sp, sp, 96
+; RV64I-NEXT: ret
+;
+; RV32M-LABEL: umulh_v2i128:
+; RV32M: # %bb.0:
+; RV32M-NEXT: addi sp, sp, -64
+; RV32M-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s8, 24(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s9, 20(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s10, 16(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sw s11, 12(sp) # 4-byte Folded Spill
+; RV32M-NEXT: lw a5, 0(a1)
+; RV32M-NEXT: lw a3, 4(a1)
+; RV32M-NEXT: lw t2, 0(a2)
+; RV32M-NEXT: lw a4, 4(a2)
+; RV32M-NEXT: lw t1, 8(a2)
+; RV32M-NEXT: lw a6, 12(a2)
+; RV32M-NEXT: lw t0, 8(a1)
+; RV32M-NEXT: lw a7, 12(a1)
+; RV32M-NEXT: mulhu t5, a5, t2
+; RV32M-NEXT: mul t6, a3, t2
+; RV32M-NEXT: mul s0, a5, a4
+; RV32M-NEXT: mulhu s1, a3, t2
+; RV32M-NEXT: mulhu s2, a5, a4
+; RV32M-NEXT: mul s3, a3, a4
+; RV32M-NEXT: mulhu s4, a3, a4
+; RV32M-NEXT: mulhu s5, t0, t2
+; RV32M-NEXT: mul t3, a7, t2
+; RV32M-NEXT: mul t4, t0, a4
+; RV32M-NEXT: add t5, t6, t5
+; RV32M-NEXT: sltu t6, t5, t6
+; RV32M-NEXT: add t5, s0, t5
+; RV32M-NEXT: sltu t5, t5, s0
+; RV32M-NEXT: mul s0, t0, t2
+; RV32M-NEXT: add t6, s1, t6
+; RV32M-NEXT: add t5, s2, t5
+; RV32M-NEXT: add t5, t6, t5
+; RV32M-NEXT: sltu t6, t5, t6
+; RV32M-NEXT: add t5, s3, t5
+; RV32M-NEXT: add t6, s4, t6
+; RV32M-NEXT: sltu s1, t5, s3
+; RV32M-NEXT: add s2, t3, s5
+; RV32M-NEXT: add s1, t6, s1
+; RV32M-NEXT: add s3, t4, s2
+; RV32M-NEXT: add t6, s0, t5
+; RV32M-NEXT: add s1, s3, s1
+; RV32M-NEXT: sltu t5, t6, s0
+; RV32M-NEXT: add s1, s1, t5
+; RV32M-NEXT: beq s1, s3, .LBB15_2
+; RV32M-NEXT: # %bb.1:
+; RV32M-NEXT: sltu t5, s1, s3
+; RV32M-NEXT: .LBB15_2:
+; RV32M-NEXT: mulhu t2, a7, t2
+; RV32M-NEXT: mulhu s6, t0, a4
+; RV32M-NEXT: mulhu s7, a5, t1
+; RV32M-NEXT: mul s4, a3, t1
+; RV32M-NEXT: mul s5, a5, a6
+; RV32M-NEXT: mul s8, a5, t1
+; RV32M-NEXT: mul s0, a7, a4
+; RV32M-NEXT: sltu t3, s2, t3
+; RV32M-NEXT: sltu s2, s3, t4
+; RV32M-NEXT: add t4, t2, t3
+; RV32M-NEXT: add s2, s6, s2
+; RV32M-NEXT: add s3, s4, s7
+; RV32M-NEXT: add s2, t4, s2
+; RV32M-NEXT: add s6, s5, s3
+; RV32M-NEXT: add t6, s8, t6
+; RV32M-NEXT: add s1, s6, s1
+; RV32M-NEXT: sltu t3, t6, s8
+; RV32M-NEXT: add t6, s0, s2
+; RV32M-NEXT: add s1, s1, t3
+; RV32M-NEXT: add t2, t6, t5
+; RV32M-NEXT: beq s1, s6, .LBB15_4
+; RV32M-NEXT: # %bb.3:
+; RV32M-NEXT: sltu t3, s1, s6
+; RV32M-NEXT: .LBB15_4:
+; RV32M-NEXT: mulhu t5, a3, t1
+; RV32M-NEXT: mulhu a5, a5, a6
+; RV32M-NEXT: sltu s1, s3, s4
+; RV32M-NEXT: sltu s3, s6, s5
+; RV32M-NEXT: mul s4, a3, a6
+; RV32M-NEXT: sltu s5, t2, t6
+; RV32M-NEXT: sltu t6, t6, s0
+; RV32M-NEXT: sltu t4, s2, t4
+; RV32M-NEXT: mulhu a4, a7, a4
+; RV32M-NEXT: mulhu a3, a3, a6
+; RV32M-NEXT: mulhu s0, t0, t1
+; RV32M-NEXT: add t5, t5, s1
+; RV32M-NEXT: add a5, a5, s3
+; RV32M-NEXT: add t4, t4, t6
+; RV32M-NEXT: add a5, t5, a5
+; RV32M-NEXT: add t4, t4, s5
+; RV32M-NEXT: add t6, s4, a5
+; RV32M-NEXT: sltu a5, a5, t5
+; RV32M-NEXT: mul s1, a7, t1
+; RV32M-NEXT: add t3, t6, t3
+; RV32M-NEXT: mul t5, t0, t1
+; RV32M-NEXT: add s6, t2, t3
+; RV32M-NEXT: mul s2, t0, a6
+; RV32M-NEXT: sltu s3, t6, s4
+; RV32M-NEXT: add a3, a3, a5
+; RV32M-NEXT: sltu a5, t3, t6
+; RV32M-NEXT: add a3, a3, s3
+; RV32M-NEXT: add s5, a4, t4
+; RV32M-NEXT: add a3, a3, a5
+; RV32M-NEXT: sltu t6, s6, t2
+; RV32M-NEXT: add a4, s5, a3
+; RV32M-NEXT: add s3, s1, s0
+; RV32M-NEXT: add s6, t5, s6
+; RV32M-NEXT: add s4, s2, s3
+; RV32M-NEXT: sw s6, 8(sp) # 4-byte Folded Spill
+; RV32M-NEXT: sltu t5, s6, t5
+; RV32M-NEXT: add s6, a4, t6
+; RV32M-NEXT: add a4, s4, t5
+; RV32M-NEXT: add a4, a4, s6
+; RV32M-NEXT: beq a4, s4, .LBB15_6
+; RV32M-NEXT: # %bb.5:
+; RV32M-NEXT: sltu t5, a4, s4
+; RV32M-NEXT: .LBB15_6:
+; RV32M-NEXT: lw t4, 16(a1)
+; RV32M-NEXT: lw t2, 20(a1)
+; RV32M-NEXT: lw a5, 24(a1)
+; RV32M-NEXT: lw a1, 28(a1)
+; RV32M-NEXT: lw s0, 16(a2)
+; RV32M-NEXT: lw t3, 20(a2)
+; RV32M-NEXT: beq s6, s5, .LBB15_8
+; RV32M-NEXT: # %bb.7:
+; RV32M-NEXT: sltu t6, s6, s5
+; RV32M-NEXT: .LBB15_8:
+; RV32M-NEXT: mulhu t1, a7, t1
+; RV32M-NEXT: mulhu t0, t0, a6
+; RV32M-NEXT: mulhu s5, t4, s0
+; RV32M-NEXT: mul s6, t2, s0
+; RV32M-NEXT: mul s7, t4, t3
+; RV32M-NEXT: sltu s1, s3, s1
+; RV32M-NEXT: sltu s2, s4, s2
+; RV32M-NEXT: mul s9, a7, a6
+; RV32M-NEXT: mulhu a6, a7, a6
+; RV32M-NEXT: mulhu a7, t2, s0
+; RV32M-NEXT: mulhu s3, t4, t3
+; RV32M-NEXT: add t1, t1, s1
+; RV32M-NEXT: add t0, t0, s2
+; RV32M-NEXT: add s5, s6, s5
+; RV32M-NEXT: add t0, t1, t0
+; RV32M-NEXT: sltu s1, s5, s6
+; RV32M-NEXT: add s5, s7, s5
+; RV32M-NEXT: sltu s2, t0, t1
+; RV32M-NEXT: mul s4, t2, t3
+; RV32M-NEXT: sltu t1, s5, s7
+; RV32M-NEXT: mulhu s5, t2, t3
+; RV32M-NEXT: add s6, a7, s1
+; RV32M-NEXT: add s3, s3, t1
+; RV32M-NEXT: mulhu s7, a5, s0
+; RV32M-NEXT: mul a7, a1, s0
+; RV32M-NEXT: mul t1, a5, t3
+; RV32M-NEXT: mul s8, a5, s0
+; RV32M-NEXT: add s1, a6, s2
+; RV32M-NEXT: add s3, s6, s3
+; RV32M-NEXT: sltu a6, s3, s6
+; RV32M-NEXT: add s3, s4, s3
+; RV32M-NEXT: add s5, s5, a6
+; RV32M-NEXT: lw a6, 24(a2)
+; RV32M-NEXT: lw a2, 28(a2)
+; RV32M-NEXT: sltu s2, s3, s4
+; RV32M-NEXT: add s2, s5, s2
+; RV32M-NEXT: add s5, a7, s7
+; RV32M-NEXT: add s6, t1, s5
+; RV32M-NEXT: add s3, s8, s3
+; RV32M-NEXT: add s4, s6, s2
+; RV32M-NEXT: sltu s2, s3, s8
+; RV32M-NEXT: add s8, s9, t0
+; RV32M-NEXT: add s4, s4, s2
+; RV32M-NEXT: add t0, s8, t6
+; RV32M-NEXT: sltu t6, s8, s9
+; RV32M-NEXT: beq s4, s6, .LBB15_10
+; RV32M-NEXT: # %bb.9:
+; RV32M-NEXT: sltu s2, s4, s6
+; RV32M-NEXT: .LBB15_10:
+; RV32M-NEXT: mulhu s0, a1, s0
+; RV32M-NEXT: mulhu s10, a5, t3
+; RV32M-NEXT: mulhu s11, t4, a6
+; RV32M-NEXT: mul s7, t2, a6
+; RV32M-NEXT: mul s9, t4, a2
+; RV32M-NEXT: mul ra, t4, a6
+; RV32M-NEXT: sltu s5, s5, a7
+; RV32M-NEXT: sltu a3, s6, t1
+; RV32M-NEXT: mul s6, a1, t3
+; RV32M-NEXT: add s5, s0, s5
+; RV32M-NEXT: add a7, t0, t5
+; RV32M-NEXT: sltu t1, t0, s8
+; RV32M-NEXT: add a3, s10, a3
+; RV32M-NEXT: add s10, s7, s11
+; RV32M-NEXT: add s8, s5, a3
+; RV32M-NEXT: add s11, s9, s10
+; RV32M-NEXT: add s3, ra, s3
+; RV32M-NEXT: add s4, s11, s4
+; RV32M-NEXT: sltu s0, s3, ra
+; RV32M-NEXT: add s3, s6, s8
+; RV32M-NEXT: add s4, s4, s0
+; RV32M-NEXT: add t5, s1, t6
+; RV32M-NEXT: add t6, s3, s2
+; RV32M-NEXT: beq s4, s11, .LBB15_12
+; RV32M-NEXT: # %bb.11:
+; RV32M-NEXT: sltu s0, s4, s11
+; RV32M-NEXT: .LBB15_12:
+; RV32M-NEXT: mulhu a3, t2, a6
+; RV32M-NEXT: mulhu t4, t4, a2
+; RV32M-NEXT: sltu s1, s10, s7
+; RV32M-NEXT: sltu s2, s11, s9
+; RV32M-NEXT: mul s4, t2, a2
+; RV32M-NEXT: sltu s6, s3, s6
+; RV32M-NEXT: sltu s3, t6, s3
+; RV32M-NEXT: sltu s5, s8, s5
+; RV32M-NEXT: mulhu t3, a1, t3
+; RV32M-NEXT: mulhu s7, t2, a2
+; RV32M-NEXT: mulhu s8, a5, a6
+; RV32M-NEXT: add a3, a3, s1
+; RV32M-NEXT: add t4, t4, s2
+; RV32M-NEXT: add s5, s5, s6
+; RV32M-NEXT: add t4, a3, t4
+; RV32M-NEXT: add s3, s5, s3
+; RV32M-NEXT: add s1, s4, t4
+; RV32M-NEXT: sltu a3, t4, a3
+; RV32M-NEXT: mul t2, a1, a6
+; RV32M-NEXT: add t4, s1, s0
+; RV32M-NEXT: mul s5, a5, a6
+; RV32M-NEXT: add s6, t6, t4
+; RV32M-NEXT: mul s0, a5, a2
+; RV32M-NEXT: sltu s2, s1, s4
+; RV32M-NEXT: add a3, s7, a3
+; RV32M-NEXT: sltu t4, t4, s1
+; RV32M-NEXT: add a3, a3, s2
+; RV32M-NEXT: add s4, t3, s3
+; RV32M-NEXT: add a3, a3, t4
+; RV32M-NEXT: sltu t6, s6, t6
+; RV32M-NEXT: add a3, s4, a3
+; RV32M-NEXT: add s2, t2, s8
+; RV32M-NEXT: add t3, s5, s6
+; RV32M-NEXT: add s3, s0, s2
+; RV32M-NEXT: sltu s1, t3, s5
+; RV32M-NEXT: add s5, a3, t6
+; RV32M-NEXT: add t4, s3, s1
+; RV32M-NEXT: add t4, t4, s5
+; RV32M-NEXT: sltu t0, a7, t0
+; RV32M-NEXT: add t1, t5, t1
+; RV32M-NEXT: beq t4, s3, .LBB15_14
+; RV32M-NEXT: # %bb.13:
+; RV32M-NEXT: sltu s1, t4, s3
+; RV32M-NEXT: .LBB15_14:
+; RV32M-NEXT: add t0, t1, t0
+; RV32M-NEXT: beq s5, s4, .LBB15_16
+; RV32M-NEXT: # %bb.15:
+; RV32M-NEXT: sltu t6, s5, s4
+; RV32M-NEXT: .LBB15_16:
+; RV32M-NEXT: mulhu a3, a1, a6
+; RV32M-NEXT: mulhu a5, a5, a2
+; RV32M-NEXT: mul a6, a1, a2
+; RV32M-NEXT: mulhu a1, a1, a2
+; RV32M-NEXT: sltu a2, s2, t2
+; RV32M-NEXT: sltu t1, s3, s0
+; RV32M-NEXT: add a2, a3, a2
+; RV32M-NEXT: add a5, a5, t1
+; RV32M-NEXT: add a5, a2, a5
+; RV32M-NEXT: add a3, a6, a5
+; RV32M-NEXT: sltu a2, a5, a2
+; RV32M-NEXT: add t6, a3, t6
+; RV32M-NEXT: sltu a5, a3, a6
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add s1, t6, s1
+; RV32M-NEXT: sltu a2, t6, a3
+; RV32M-NEXT: add a1, a1, a5
+; RV32M-NEXT: sltu a3, s1, t6
+; RV32M-NEXT: add a1, a1, a2
+; RV32M-NEXT: add a1, a1, a3
+; RV32M-NEXT: sw t3, 16(a0)
+; RV32M-NEXT: sw t4, 20(a0)
+; RV32M-NEXT: sw s1, 24(a0)
+; RV32M-NEXT: sw a1, 28(a0)
+; RV32M-NEXT: lw a1, 8(sp) # 4-byte Folded Reload
+; RV32M-NEXT: sw a1, 0(a0)
+; RV32M-NEXT: sw a4, 4(a0)
+; RV32M-NEXT: sw a7, 8(a0)
+; RV32M-NEXT: sw t0, 12(a0)
+; RV32M-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s8, 24(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s9, 20(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s10, 16(sp) # 4-byte Folded Reload
+; RV32M-NEXT: lw s11, 12(sp) # 4-byte Folded Reload
+; RV32M-NEXT: addi sp, sp, 64
+; RV32M-NEXT: ret
+;
+; RV64M-LABEL: umulh_v2i128:
+; RV64M: # %bb.0:
+; RV64M-NEXT: ld a3, 0(a1)
+; RV64M-NEXT: ld a4, 8(a1)
+; RV64M-NEXT: ld a5, 0(a2)
+; RV64M-NEXT: ld a6, 8(a2)
+; RV64M-NEXT: ld a7, 16(a2)
+; RV64M-NEXT: ld a2, 24(a2)
+; RV64M-NEXT: ld t0, 16(a1)
+; RV64M-NEXT: ld a1, 24(a1)
+; RV64M-NEXT: mulhu t1, a3, a5
+; RV64M-NEXT: mul t2, a4, a5
+; RV64M-NEXT: mulhu a5, a4, a5
+; RV64M-NEXT: mul t3, a3, a6
+; RV64M-NEXT: mulhu a3, a3, a6
+; RV64M-NEXT: mul t4, a4, a6
+; RV64M-NEXT: mulhu a4, a4, a6
+; RV64M-NEXT: add t1, t2, t1
+; RV64M-NEXT: mulhu a6, t0, a7
+; RV64M-NEXT: sltu t2, t1, t2
+; RV64M-NEXT: mul t5, a1, a7
+; RV64M-NEXT: add a5, a5, t2
+; RV64M-NEXT: mul t2, t0, a2
+; RV64M-NEXT: add t1, t3, t1
+; RV64M-NEXT: sltu t1, t1, t3
+; RV64M-NEXT: mulhu a7, a1, a7
+; RV64M-NEXT: mulhu t0, t0, a2
+; RV64M-NEXT: add a3, a3, t1
+; RV64M-NEXT: mul t1, a1, a2
+; RV64M-NEXT: add a3, a5, a3
+; RV64M-NEXT: mulhu a1, a1, a2
+; RV64M-NEXT: sltu a2, a3, a5
+; RV64M-NEXT: add a3, t4, a3
+; RV64M-NEXT: add a6, t5, a6
+; RV64M-NEXT: sltu a5, a6, t5
+; RV64M-NEXT: add a6, t2, a6
+; RV64M-NEXT: sltu a6, a6, t2
+; RV64M-NEXT: sltu t2, a3, t4
+; RV64M-NEXT: add a5, a7, a5
+; RV64M-NEXT: add a6, t0, a6
+; RV64M-NEXT: add a2, a4, a2
+; RV64M-NEXT: add a6, a5, a6
+; RV64M-NEXT: add a4, t1, a6
+; RV64M-NEXT: sltu a5, a6, a5
+; RV64M-NEXT: sltu a6, a4, t1
+; RV64M-NEXT: add a1, a1, a5
+; RV64M-NEXT: add a2, a2, t2
+; RV64M-NEXT: add a1, a1, a6
+; RV64M-NEXT: sd a3, 0(a0)
+; RV64M-NEXT: sd a2, 8(a0)
+; RV64M-NEXT: sd a4, 16(a0)
+; RV64M-NEXT: sd a1, 24(a0)
+; RV64M-NEXT: ret
+ %r = call <2 x i128> @llvm.umulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; CHECK-I: {{.*}}
diff --git a/llvm/test/CodeGen/X86/smulh.ll b/llvm/test/CodeGen/X86/smulh.ll
new file mode 100644
index 0000000000000..d132778c0faa0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/smulh.ll
@@ -0,0 +1,851 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+
+define i16 @smulh_i16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: smulh_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movswl %si, %ecx
+; CHECK-NEXT: movswl %di, %eax
+; CHECK-NEXT: imull %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %r = call i16 @llvm.smulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @smulh_i32(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: smulh_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movslq %esi, %rcx
+; CHECK-NEXT: movslq %edi, %rax
+; CHECK-NEXT: imulq %rcx, %rax
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+ %r = call i32 @llvm.smulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @smulh_i64(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: smulh_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: imulq %rsi
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: retq
+ %r = call i64 @llvm.smulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @smulh_i96(i96 %x, i96 %y) nounwind {
+; SSE-LABEL: smulh_i96:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %r8
+; SSE-NEXT: movslq %ecx, %r10
+; SSE-NEXT: shldq $32, %rdi, %rsi
+; SSE-NEXT: movq %rsi, %r11
+; SSE-NEXT: sarq $63, %r11
+; SSE-NEXT: shlq $32, %rdi
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rdx
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r8
+; SSE-NEXT: movq %rax, %r9
+; SSE-NEXT: imulq %r8, %r11
+; SSE-NEXT: addq %rcx, %r9
+; SSE-NEXT: adcq %rdx, %r11
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: sarq $63, %rcx
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: sarq $63, %r8
+; SSE-NEXT: imulq %rdi, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %r10
+; SSE-NEXT: addq %r9, %rax
+; SSE-NEXT: adcq %rdx, %r8
+; SSE-NEXT: movq %r8, %rdi
+; SSE-NEXT: sarq $63, %rdi
+; SSE-NEXT: addq %r11, %r8
+; SSE-NEXT: adcq %rcx, %rdi
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: imulq %r10
+; SSE-NEXT: addq %r8, %rax
+; SSE-NEXT: adcq %rdi, %rdx
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: smulh_i96:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rax
+; AVX2-NEXT: shldq $32, %rdi, %rax
+; AVX2-NEXT: shlq $32, %rdi
+; AVX2-NEXT: mulxq %rdi, %rsi, %rsi
+; AVX2-NEXT: mulxq %rax, %r8, %r9
+; AVX2-NEXT: movq %rax, %r10
+; AVX2-NEXT: sarq $63, %r10
+; AVX2-NEXT: imulq %rdx, %r10
+; AVX2-NEXT: addq %rsi, %r8
+; AVX2-NEXT: adcq %r10, %r9
+; AVX2-NEXT: movslq %ecx, %rcx
+; AVX2-NEXT: movq %r9, %rsi
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rcx, %r10
+; AVX2-NEXT: sarq $63, %r10
+; AVX2-NEXT: imulq %rdi, %r10
+; AVX2-NEXT: movq %rdi, %rdx
+; AVX2-NEXT: mulxq %rcx, %rdx, %rdi
+; AVX2-NEXT: addq %r8, %rdx
+; AVX2-NEXT: adcq %r10, %rdi
+; AVX2-NEXT: movq %rdi, %r8
+; AVX2-NEXT: sarq $63, %r8
+; AVX2-NEXT: addq %r9, %rdi
+; AVX2-NEXT: adcq %rsi, %r8
+; AVX2-NEXT: imulq %rcx
+; AVX2-NEXT: addq %rdi, %rax
+; AVX2-NEXT: adcq %r8, %rdx
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_i96:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rsi, %rax
+; AVX512-NEXT: movslq %ecx, %rcx
+; AVX512-NEXT: shldq $32, %rdi, %rax
+; AVX512-NEXT: shlq $32, %rdi
+; AVX512-NEXT: mulxq %rdi, %rsi, %rsi
+; AVX512-NEXT: mulxq %rax, %r8, %r9
+; AVX512-NEXT: movq %rax, %r10
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: imulq %rdx, %r10
+; AVX512-NEXT: addq %rsi, %r8
+; AVX512-NEXT: adcq %r10, %r9
+; AVX512-NEXT: movq %r9, %rsi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: movq %rcx, %r10
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: imulq %rdi, %r10
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: mulxq %rcx, %rdx, %rdi
+; AVX512-NEXT: addq %r8, %rdx
+; AVX512-NEXT: adcq %r10, %rdi
+; AVX512-NEXT: movq %rdi, %r8
+; AVX512-NEXT: sarq $63, %r8
+; AVX512-NEXT: addq %r9, %rdi
+; AVX512-NEXT: adcq %rsi, %r8
+; AVX512-NEXT: imulq %rcx
+; AVX512-NEXT: addq %rdi, %rax
+; AVX512-NEXT: adcq %r8, %rdx
+; AVX512-NEXT: retq
+ %r = call i96 @llvm.smulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @smulh_i128(i128 %x, i128 %y) nounwind {
+; SSE-LABEL: smulh_i128:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %r8
+; SSE-NEXT: movq %rsi, %r11
+; SSE-NEXT: sarq $63, %r11
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rdx
+; SSE-NEXT: movq %rdx, %r9
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r8
+; SSE-NEXT: movq %rax, %r10
+; SSE-NEXT: imulq %r8, %r11
+; SSE-NEXT: addq %r9, %r10
+; SSE-NEXT: adcq %rdx, %r11
+; SSE-NEXT: movq %r11, %r8
+; SSE-NEXT: sarq $63, %r8
+; SSE-NEXT: movq %rcx, %r9
+; SSE-NEXT: sarq $63, %r9
+; SSE-NEXT: imulq %rdi, %r9
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rcx
+; SSE-NEXT: addq %r10, %rax
+; SSE-NEXT: adcq %rdx, %r9
+; SSE-NEXT: movq %r9, %rdi
+; SSE-NEXT: sarq $63, %rdi
+; SSE-NEXT: addq %r11, %r9
+; SSE-NEXT: adcq %r8, %rdi
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: imulq %rcx
+; SSE-NEXT: addq %r9, %rax
+; SSE-NEXT: adcq %rdi, %rdx
+; SSE-NEXT: retq
+;
+; AVX-LABEL: smulh_i128:
+; AVX: # %bb.0:
+; AVX-NEXT: movq %rsi, %rax
+; AVX-NEXT: mulxq %rdi, %rsi, %rsi
+; AVX-NEXT: mulxq %rax, %r8, %r9
+; AVX-NEXT: movq %rax, %r10
+; AVX-NEXT: sarq $63, %r10
+; AVX-NEXT: imulq %rdx, %r10
+; AVX-NEXT: addq %rsi, %r8
+; AVX-NEXT: adcq %r10, %r9
+; AVX-NEXT: movq %r9, %rsi
+; AVX-NEXT: sarq $63, %rsi
+; AVX-NEXT: movq %rcx, %r10
+; AVX-NEXT: sarq $63, %r10
+; AVX-NEXT: imulq %rdi, %r10
+; AVX-NEXT: movq %rdi, %rdx
+; AVX-NEXT: mulxq %rcx, %rdx, %rdi
+; AVX-NEXT: addq %r8, %rdx
+; AVX-NEXT: adcq %r10, %rdi
+; AVX-NEXT: movq %rdi, %r8
+; AVX-NEXT: sarq $63, %r8
+; AVX-NEXT: addq %r9, %rdi
+; AVX-NEXT: adcq %rsi, %r8
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: addq %rdi, %rax
+; AVX-NEXT: adcq %r8, %rdx
+; AVX-NEXT: retq
+ %r = call i128 @llvm.smulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @smulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; SSE-LABEL: smulh_v8i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE-NEXT: pxor %xmm4, %xmm4
+; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE-NEXT: pmulhw %xmm3, %xmm4
+; SSE-NEXT: psrlw $8, %xmm4
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE-NEXT: pmulhw %xmm3, %xmm2
+; SSE-NEXT: psrlw $8, %xmm2
+; SSE-NEXT: packuswb %xmm4, %xmm2
+; SSE-NEXT: movdqa %xmm2, %xmm0
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: smulh_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %r = call <8 x i8> @llvm.smulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @smulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; SSE-LABEL: smulh_v4i16:
+; SSE: # %bb.0:
+; SSE-NEXT: pmulhw %xmm1, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: smulh_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmulhw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %r = call <4 x i16> @llvm.smulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @smulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; SSE2-LABEL: smulh_v2i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: smulh_v2i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuldq %xmm2, %xmm3
+; SSE42-NEXT: pmuldq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: smulh_v2i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v2i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <2 x i32> @llvm.smulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @smulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; CHECK-LABEL: smulh_v1i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: imulq %rsi
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: retq
+ %r = call <1 x i64> @llvm.smulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @smulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; SSE-LABEL: smulh_v16i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE-NEXT: pxor %xmm4, %xmm4
+; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
+; SSE-NEXT: pmulhw %xmm3, %xmm4
+; SSE-NEXT: psrlw $8, %xmm4
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE-NEXT: pmulhw %xmm3, %xmm2
+; SSE-NEXT: psrlw $8, %xmm2
+; SSE-NEXT: packuswb %xmm4, %xmm2
+; SSE-NEXT: movdqa %xmm2, %xmm0
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: smulh_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %r = call <16 x i8> @llvm.smulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @smulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; SSE-LABEL: smulh_v8i16:
+; SSE: # %bb.0:
+; SSE-NEXT: pmulhw %xmm1, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: smulh_v8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmulhw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %r = call <8 x i16> @llvm.smulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @smulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; SSE2-LABEL: smulh_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: smulh_v4i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuldq %xmm2, %xmm3
+; SSE42-NEXT: pmuldq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: smulh_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <4 x i32> @llvm.smulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @smulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: smulh_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: imulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rcx
+; SSE2-NEXT: imulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: smulh_v2i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm1, %rcx
+; SSE42-NEXT: imulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm2
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: movq %xmm1, %rcx
+; SSE42-NEXT: imulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE42-NEXT: retq
+;
+; AVX-LABEL: smulh_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm2
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vmovq %xmm1, %rcx
+; AVX-NEXT: imulq %rcx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX-NEXT: retq
+ %r = call <2 x i64> @llvm.smulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @smulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; SSE2-LABEL: smulh_v3i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: smulh_v3i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuldq %xmm2, %xmm3
+; SSE42-NEXT: pmuldq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: smulh_v3i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <3 x i32> @llvm.smulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @smulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; SSE2-LABEL: smulh_v2i48:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; SSE2-NEXT: psllq $16, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: psllq $16, %xmm0
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: imulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: imulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: smulh_v2i48:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm1, %xmm2
+; SSE42-NEXT: psllq $16, %xmm2
+; SSE42-NEXT: psrad $16, %xmm2
+; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE42-NEXT: pextrq $1, %xmm2, %rcx
+; SSE42-NEXT: psllq $16, %xmm0
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: imulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm1
+; SSE42-NEXT: movq %xmm2, %rcx
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: imulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: smulh_v2i48:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpsllq $16, %xmm1, %xmm2
+; AVX2-NEXT: vpsrad $16, %xmm2, %xmm2
+; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX2-NEXT: vpsllq $16, %xmm0, %xmm0
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: imulq %rcx
+; AVX2-NEXT: vmovq %rdx, %xmm2
+; AVX2-NEXT: vmovq %xmm1, %rcx
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: imulq %rcx
+; AVX2-NEXT: vmovq %rdx, %xmm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v2i48:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllq $16, %xmm1, %xmm1
+; AVX512-NEXT: vpsllq $16, %xmm0, %xmm0
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: vpsraq $16, %xmm1, %xmm1
+; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512-NEXT: imulq %rcx
+; AVX512-NEXT: vmovq %rdx, %xmm2
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vmovq %xmm1, %rcx
+; AVX512-NEXT: imulq %rcx
+; AVX512-NEXT: vmovq %rdx, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512-NEXT: retq
+ %r = call <2 x i48> @llvm.smulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @smulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; SSE-LABEL: smulh_v2i128:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE-NEXT: movq %rdx, %r12
+; SSE-NEXT: sarq $63, %r12
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r9
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %r10, %rax
+; SSE-NEXT: mulq %r9
+; SSE-NEXT: movq %rax, %rbx
+; SSE-NEXT: movq %rdx, %r14
+; SSE-NEXT: imulq %r9, %r12
+; SSE-NEXT: addq %r11, %rbx
+; SSE-NEXT: adcq %r12, %r14
+; SSE-NEXT: movq %r14, %r9
+; SSE-NEXT: sarq $63, %r9
+; SSE-NEXT: movq %r15, %r12
+; SSE-NEXT: sarq $63, %r12
+; SSE-NEXT: imulq %rsi, %r12
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r15
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: addq %rbx, %rax
+; SSE-NEXT: adcq %r12, %r11
+; SSE-NEXT: movq %r11, %rbx
+; SSE-NEXT: sarq $63, %rbx
+; SSE-NEXT: addq %r14, %r11
+; SSE-NEXT: adcq %r9, %rbx
+; SSE-NEXT: movq %r10, %rax
+; SSE-NEXT: imulq %r15
+; SSE-NEXT: movq %rdx, %rsi
+; SSE-NEXT: movq %rax, %r9
+; SSE-NEXT: addq %r11, %r9
+; SSE-NEXT: adcq %rbx, %rsi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq %r8, %r14
+; SSE-NEXT: sarq $63, %r14
+; SSE-NEXT: imulq %r10, %r14
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: mulq %r10
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %r8, %rax
+; SSE-NEXT: mulq %r10
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rax, %rbx
+; SSE-NEXT: addq %r11, %rbx
+; SSE-NEXT: adcq %r14, %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: movq %r10, %r15
+; SSE-NEXT: sarq $63, %r15
+; SSE-NEXT: movq %r14, %r12
+; SSE-NEXT: sarq $63, %r12
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: mulq %r14
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: imulq %rcx, %r12
+; SSE-NEXT: addq %rbx, %rax
+; SSE-NEXT: adcq %r12, %r11
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: sarq $63, %rcx
+; SSE-NEXT: addq %r10, %r11
+; SSE-NEXT: adcq %r15, %rcx
+; SSE-NEXT: movq %r8, %rax
+; SSE-NEXT: imulq %r14
+; SSE-NEXT: addq %r11, %rax
+; SSE-NEXT: adcq %rcx, %rdx
+; SSE-NEXT: movq %rax, 16(%rdi)
+; SSE-NEXT: movq %r9, (%rdi)
+; SSE-NEXT: movq %rdx, 24(%rdi)
+; SSE-NEXT: movq %rsi, 8(%rdi)
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: smulh_v2i128:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdx, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq %rdx, %r11
+; AVX2-NEXT: sarq $63, %r11
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %r9, %rbx, %rbx
+; AVX2-NEXT: imulq %r9, %r11
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: mulxq %r9, %r9, %r14
+; AVX2-NEXT: addq %rbx, %r9
+; AVX2-NEXT: adcq %r11, %r14
+; AVX2-NEXT: movq %r14, %r11
+; AVX2-NEXT: sarq $63, %r11
+; AVX2-NEXT: movq %r10, %rbx
+; AVX2-NEXT: sarq $63, %rbx
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %r10, %rdx, %r15
+; AVX2-NEXT: imulq %rsi, %rbx
+; AVX2-NEXT: addq %r9, %rdx
+; AVX2-NEXT: adcq %rbx, %r15
+; AVX2-NEXT: movq %r15, %rbx
+; AVX2-NEXT: sarq $63, %rbx
+; AVX2-NEXT: addq %r14, %r15
+; AVX2-NEXT: adcq %r11, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: imulq %r10
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: movq %rax, %r9
+; AVX2-NEXT: addq %r15, %r9
+; AVX2-NEXT: adcq %rbx, %rsi
+; AVX2-NEXT: movq %r8, %rax
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: mulxq %r11, %r10, %r10
+; AVX2-NEXT: sarq $63, %rax
+; AVX2-NEXT: movq %r8, %rdx
+; AVX2-NEXT: mulxq %r11, %rbx, %r14
+; AVX2-NEXT: imulq %r11, %rax
+; AVX2-NEXT: addq %r10, %rbx
+; AVX2-NEXT: adcq %rax, %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq %r14, %rax
+; AVX2-NEXT: sarq $63, %rax
+; AVX2-NEXT: movq %r10, %r11
+; AVX2-NEXT: sarq $63, %r11
+; AVX2-NEXT: imulq %rcx, %r11
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: mulxq %r10, %rcx, %r15
+; AVX2-NEXT: addq %rbx, %rcx
+; AVX2-NEXT: adcq %r11, %r15
+; AVX2-NEXT: movq %r15, %rcx
+; AVX2-NEXT: sarq $63, %rcx
+; AVX2-NEXT: addq %r14, %r15
+; AVX2-NEXT: adcq %rax, %rcx
+; AVX2-NEXT: movq %r8, %rax
+; AVX2-NEXT: imulq %r10
+; AVX2-NEXT: addq %r15, %rax
+; AVX2-NEXT: adcq %rcx, %rdx
+; AVX2-NEXT: movq %rax, 16(%rdi)
+; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rdx, 24(%rdi)
+; AVX2-NEXT: movq %rsi, 8(%rdi)
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: smulh_v2i128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %r13
+; AVX512-NEXT: pushq %r12
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdx, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: movq %rdx, %r14
+; AVX512-NEXT: sarq $63, %r14
+; AVX512-NEXT: imulq %r9, %r14
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %r9, %r15, %r15
+; AVX512-NEXT: movq %rax, %rdx
+; AVX512-NEXT: mulxq %r9, %r9, %r12
+; AVX512-NEXT: addq %r15, %r9
+; AVX512-NEXT: adcq %r14, %r12
+; AVX512-NEXT: movq %r12, %r14
+; AVX512-NEXT: sarq $63, %r14
+; AVX512-NEXT: movq %rbx, %r15
+; AVX512-NEXT: sarq $63, %r15
+; AVX512-NEXT: imulq %rsi, %r15
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %rbx, %rdx, %r13
+; AVX512-NEXT: addq %r9, %rdx
+; AVX512-NEXT: adcq %r15, %r13
+; AVX512-NEXT: movq %r13, %r15
+; AVX512-NEXT: sarq $63, %r15
+; AVX512-NEXT: addq %r12, %r13
+; AVX512-NEXT: adcq %r14, %r15
+; AVX512-NEXT: imulq %rbx
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: movq %rax, %r9
+; AVX512-NEXT: addq %r13, %r9
+; AVX512-NEXT: adcq %r15, %rsi
+; AVX512-NEXT: movq %r8, %rax
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: imulq %r11, %rax
+; AVX512-NEXT: movq %rcx, %rdx
+; AVX512-NEXT: mulxq %r11, %rbx, %rbx
+; AVX512-NEXT: movq %r8, %rdx
+; AVX512-NEXT: mulxq %r11, %r11, %r14
+; AVX512-NEXT: addq %rbx, %r11
+; AVX512-NEXT: adcq %rax, %r14
+; AVX512-NEXT: movq %r14, %rax
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %r10, %rbx
+; AVX512-NEXT: sarq $63, %rbx
+; AVX512-NEXT: imulq %rcx, %rbx
+; AVX512-NEXT: movq %rcx, %rdx
+; AVX512-NEXT: mulxq %r10, %rcx, %r15
+; AVX512-NEXT: addq %r11, %rcx
+; AVX512-NEXT: adcq %rbx, %r15
+; AVX512-NEXT: movq %r15, %rcx
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: addq %r14, %r15
+; AVX512-NEXT: adcq %rax, %rcx
+; AVX512-NEXT: movq %r8, %rax
+; AVX512-NEXT: imulq %r10
+; AVX512-NEXT: addq %r15, %rax
+; AVX512-NEXT: adcq %rcx, %rdx
+; AVX512-NEXT: movq %rax, 16(%rdi)
+; AVX512-NEXT: movq %r9, (%rdi)
+; AVX512-NEXT: movq %rdx, 24(%rdi)
+; AVX512-NEXT: movq %rsi, 8(%rdi)
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r12
+; AVX512-NEXT: popq %r13
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: retq
+ %r = call <2 x i128> @llvm.smulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
diff --git a/llvm/test/CodeGen/X86/umulh.ll b/llvm/test/CodeGen/X86/umulh.ll
new file mode 100644
index 0000000000000..766b176743d7f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/umulh.ll
@@ -0,0 +1,746 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+
+define i16 @umulh_i16(i16 %x, i16 %y) nounwind {
+; CHECK-LABEL: umulh_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzwl %si, %ecx
+; CHECK-NEXT: movzwl %di, %eax
+; CHECK-NEXT: imull %ecx, %eax
+; CHECK-NEXT: shrl $16, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %r = call i16 @llvm.umulh(i16 %x, i16 %y)
+ ret i16 %r
+}
+
+define i32 @umulh_i32(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: umulh_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %ecx
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: imulq %rcx, %rax
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+ %r = call i32 @llvm.umulh(i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @umulh_i64(i64 %x, i64 %y) nounwind {
+; SSE-LABEL: umulh_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rsi
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: umulh_i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movq %rdi, %rdx
+; AVX-NEXT: mulxq %rsi, %rax, %rax
+; AVX-NEXT: retq
+ %r = call i64 @llvm.umulh(i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define i96 @umulh_i96(i96 %x, i96 %y) nounwind {
+; SSE-LABEL: umulh_i96:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %r8
+; SSE-NEXT: movl %ecx, %r10d
+; SSE-NEXT: shldq $32, %rdi, %rsi
+; SSE-NEXT: shlq $32, %rdi
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rdx
+; SSE-NEXT: movq %rdx, %r9
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r8
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movq %rax, %r8
+; SSE-NEXT: addq %r9, %r8
+; SSE-NEXT: adcq $0, %rcx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %r10
+; SSE-NEXT: addq %r8, %rax
+; SSE-NEXT: adcq %rdx, %rcx
+; SSE-NEXT: setb %dil
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r10
+; SSE-NEXT: movzbl %dil, %esi
+; SSE-NEXT: addq %rcx, %rax
+; SSE-NEXT: adcq %rsi, %rdx
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: umulh_i96:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shlq $32, %rax
+; AVX2-NEXT: mulxq %rax, %r8, %r8
+; AVX2-NEXT: shldq $32, %rdi, %rsi
+; AVX2-NEXT: mulxq %rsi, %rdi, %r9
+; AVX2-NEXT: addq %r8, %rdi
+; AVX2-NEXT: adcq $0, %r9
+; AVX2-NEXT: movl %ecx, %ecx
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: mulxq %rcx, %rax, %r8
+; AVX2-NEXT: addq %rdi, %rax
+; AVX2-NEXT: adcq %r9, %r8
+; AVX2-NEXT: setb %dil
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %rcx, %rax, %rdx
+; AVX2-NEXT: movzbl %dil, %ecx
+; AVX2-NEXT: addq %r8, %rax
+; AVX2-NEXT: adcq %rcx, %rdx
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_i96:
+; AVX512: # %bb.0:
+; AVX512-NEXT: shldq $32, %rdi, %rsi
+; AVX512-NEXT: shlq $32, %rdi
+; AVX512-NEXT: mulxq %rdi, %rax, %rax
+; AVX512-NEXT: movl %ecx, %ecx
+; AVX512-NEXT: mulxq %rsi, %r8, %r9
+; AVX512-NEXT: addq %rax, %r8
+; AVX512-NEXT: adcq $0, %r9
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: mulxq %rcx, %rax, %rdi
+; AVX512-NEXT: addq %r8, %rax
+; AVX512-NEXT: adcq %r9, %rdi
+; AVX512-NEXT: setb %al
+; AVX512-NEXT: movzbl %al, %r8d
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %rcx, %rax, %rdx
+; AVX512-NEXT: addq %rdi, %rax
+; AVX512-NEXT: adcq %r8, %rdx
+; AVX512-NEXT: retq
+ %r = call i96 @llvm.umulh(i96 %x, i96 %y)
+ ret i96 %r
+}
+
+define i128 @umulh_i128(i128 %x, i128 %y) nounwind {
+; SSE-LABEL: umulh_i128:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %r8
+; SSE-NEXT: movq %rdi, %r9
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rdx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r8
+; SSE-NEXT: movq %rdx, %rdi
+; SSE-NEXT: movq %rax, %r8
+; SSE-NEXT: addq %r10, %r8
+; SSE-NEXT: adcq $0, %rdi
+; SSE-NEXT: movq %r9, %rax
+; SSE-NEXT: mulq %rcx
+; SSE-NEXT: addq %r8, %rax
+; SSE-NEXT: adcq %rdx, %rdi
+; SSE-NEXT: setb %r8b
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %rcx
+; SSE-NEXT: movzbl %r8b, %ecx
+; SSE-NEXT: addq %rdi, %rax
+; SSE-NEXT: adcq %rcx, %rdx
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: umulh_i128:
+; AVX2: # %bb.0:
+; AVX2-NEXT: mulxq %rdi, %rax, %rax
+; AVX2-NEXT: mulxq %rsi, %r8, %r9
+; AVX2-NEXT: addq %rax, %r8
+; AVX2-NEXT: adcq $0, %r9
+; AVX2-NEXT: movq %rdi, %rdx
+; AVX2-NEXT: mulxq %rcx, %rax, %rdi
+; AVX2-NEXT: addq %r8, %rax
+; AVX2-NEXT: adcq %r9, %rdi
+; AVX2-NEXT: setb %r8b
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %rcx, %rax, %rdx
+; AVX2-NEXT: movzbl %r8b, %ecx
+; AVX2-NEXT: addq %rdi, %rax
+; AVX2-NEXT: adcq %rcx, %rdx
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_i128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: mulxq %rdi, %rax, %rax
+; AVX512-NEXT: mulxq %rsi, %r8, %r9
+; AVX512-NEXT: addq %rax, %r8
+; AVX512-NEXT: adcq $0, %r9
+; AVX512-NEXT: movq %rdi, %rdx
+; AVX512-NEXT: mulxq %rcx, %rax, %rdi
+; AVX512-NEXT: addq %r8, %rax
+; AVX512-NEXT: adcq %r9, %rdi
+; AVX512-NEXT: setb %al
+; AVX512-NEXT: movzbl %al, %r8d
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %rcx, %rax, %rdx
+; AVX512-NEXT: addq %rdi, %rax
+; AVX512-NEXT: adcq %r8, %rdx
+; AVX512-NEXT: retq
+ %r = call i128 @llvm.umulh(i128 %x, i128 %y)
+ ret i128 %r
+}
+
+define <8 x i8> @umulh_v8i8(<8 x i8> %x, <8 x i8> %y) nounwind {
+; SSE2-LABEL: umulh_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; SSE2-NEXT: pmullw %xmm3, %xmm4
+; SSE2-NEXT: psrlw $8, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: pmullw %xmm1, %xmm0
+; SSE2-NEXT: psrlw $8, %xmm0
+; SSE2-NEXT: packuswb %xmm4, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v8i8:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE42-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE42-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
+; SSE42-NEXT: pmullw %xmm1, %xmm0
+; SSE42-NEXT: psrlw $8, %xmm0
+; SSE42-NEXT: pmullw %xmm4, %xmm2
+; SSE42-NEXT: psrlw $8, %xmm2
+; SSE42-NEXT: packuswb %xmm0, %xmm2
+; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: umulh_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %r = call <8 x i8> @llvm.umulh(<8 x i8> %x, <8 x i8> %y)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @umulh_v4i16(<4 x i16> %x, <4 x i16> %y) nounwind {
+; SSE-LABEL: umulh_v4i16:
+; SSE: # %bb.0:
+; SSE-NEXT: pmulhuw %xmm1, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: umulh_v4i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmulhuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %r = call <4 x i16> @llvm.umulh(<4 x i16> %x, <4 x i16> %y)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @umulh_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; SSE2-LABEL: umulh_v2i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v2i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuludq %xmm2, %xmm3
+; SSE42-NEXT: pmuludq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: umulh_v2i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v2i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <2 x i32> @llvm.umulh(<2 x i32> %x, <2 x i32> %y)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @umulh_v1i64(<1 x i64> %x, <1 x i64> %y) nounwind {
+; SSE-LABEL: umulh_v1i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: mulq %rsi
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: umulh_v1i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movq %rdi, %rdx
+; AVX-NEXT: mulxq %rsi, %rax, %rax
+; AVX-NEXT: retq
+ %r = call <1 x i64> @llvm.umulh(<1 x i64> %x, <1 x i64> %y)
+ ret <1 x i64> %r
+}
+
+define <16 x i8> @umulh_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
+; SSE2-LABEL: umulh_v16i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; SSE2-NEXT: pmullw %xmm3, %xmm4
+; SSE2-NEXT: psrlw $8, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: pmullw %xmm1, %xmm0
+; SSE2-NEXT: psrlw $8, %xmm0
+; SSE2-NEXT: packuswb %xmm4, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v16i8:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE42-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE42-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
+; SSE42-NEXT: pmullw %xmm1, %xmm0
+; SSE42-NEXT: psrlw $8, %xmm0
+; SSE42-NEXT: pmullw %xmm4, %xmm2
+; SSE42-NEXT: psrlw $8, %xmm2
+; SSE42-NEXT: packuswb %xmm0, %xmm2
+; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: umulh_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %r = call <16 x i8> @llvm.umulh(<16 x i8> %x, <16 x i8> %y)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @umulh_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
+; SSE-LABEL: umulh_v8i16:
+; SSE: # %bb.0:
+; SSE-NEXT: pmulhuw %xmm1, %xmm0
+; SSE-NEXT: retq
+;
+; AVX-LABEL: umulh_v8i16:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmulhuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
+ %r = call <8 x i16> @llvm.umulh(<8 x i16> %x, <8 x i16> %y)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @umulh_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; SSE2-LABEL: umulh_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v4i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuludq %xmm2, %xmm3
+; SSE42-NEXT: pmuludq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: umulh_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <4 x i32> @llvm.umulh(<4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @umulh_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: umulh_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: mulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rcx
+; SSE2-NEXT: mulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v2i64:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm1, %rcx
+; SSE42-NEXT: mulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm2
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: movq %xmm1, %rcx
+; SSE42-NEXT: mulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE42-NEXT: retq
+;
+; AVX-LABEL: umulh_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX-NEXT: vpextrq $1, %xmm1, %rax
+; AVX-NEXT: mulxq %rax, %rax, %rax
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vmovq %xmm0, %rdx
+; AVX-NEXT: vmovq %xmm1, %rax
+; AVX-NEXT: mulxq %rax, %rax, %rax
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX-NEXT: retq
+ %r = call <2 x i64> @llvm.umulh(<2 x i64> %x, <2 x i64> %y)
+ ret <2 x i64> %r
+}
+
+define <3 x i32> @umulh_v3i32(<3 x i32> %x, <3 x i32> %y) nounwind {
+; SSE2-LABEL: umulh_v3i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v3i32:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE42-NEXT: pmuludq %xmm2, %xmm3
+; SSE42-NEXT: pmuludq %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: umulh_v3i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v3i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX512-NEXT: vpmuludq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vmovdqa {{.*#+}} xmm0 = [1,5,3,7]
+; AVX512-NEXT: vpermi2d %xmm1, %xmm2, %xmm0
+; AVX512-NEXT: retq
+ %r = call <3 x i32> @llvm.umulh(<3 x i32> %x, <3 x i32> %y)
+ ret <3 x i32> %r
+}
+
+define <2 x i48> @umulh_v2i48(<2 x i48> %x, <2 x i48> %y) nounwind {
+; SSE2-LABEL: umulh_v2i48:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: psllq $16, %xmm0
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: movq %xmm1, %rcx
+; SSE2-NEXT: mulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rcx
+; SSE2-NEXT: mulq %rcx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: umulh_v2i48:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0,1,2],xmm2[3],xmm1[4,5,6],xmm2[7]
+; SSE42-NEXT: pextrq $1, %xmm2, %rcx
+; SSE42-NEXT: psllq $16, %xmm0
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: mulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm1
+; SSE42-NEXT: movq %xmm2, %rcx
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: mulq %rcx
+; SSE42-NEXT: movq %rdx, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE42-NEXT: retq
+;
+; AVX-LABEL: umulh_v2i48:
+; AVX: # %bb.0:
+; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5,6],xmm2[7]
+; AVX-NEXT: vpextrq $1, %xmm1, %rax
+; AVX-NEXT: vpsllq $16, %xmm0, %xmm0
+; AVX-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX-NEXT: mulxq %rax, %rax, %rax
+; AVX-NEXT: vmovq %rax, %xmm2
+; AVX-NEXT: vmovq %xmm1, %rax
+; AVX-NEXT: vmovq %xmm0, %rdx
+; AVX-NEXT: mulxq %rax, %rax, %rax
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX-NEXT: retq
+ %r = call <2 x i48> @llvm.umulh(<2 x i48> %x, <2 x i48> %y)
+ ret <2 x i48> %r
+}
+
+define <2 x i128> @umulh_v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
+; SSE-LABEL: umulh_v2i128:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r9
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %r10, %rax
+; SSE-NEXT: mulq %r9
+; SSE-NEXT: movq %rdx, %r9
+; SSE-NEXT: movq %rax, %rbx
+; SSE-NEXT: addq %r11, %rbx
+; SSE-NEXT: adcq $0, %r9
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: mulq %r12
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: addq %rbx, %rax
+; SSE-NEXT: adcq %r9, %r11
+; SSE-NEXT: setb %al
+; SSE-NEXT: movzbl %al, %ebx
+; SSE-NEXT: movq %r10, %rax
+; SSE-NEXT: mulq %r12
+; SSE-NEXT: movq %rdx, %rsi
+; SSE-NEXT: movq %rax, %r9
+; SSE-NEXT: addq %r11, %r9
+; SSE-NEXT: adcq %rbx, %rsi
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: mulq %r15
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %r8, %rax
+; SSE-NEXT: mulq %r15
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %rax, %rbx
+; SSE-NEXT: addq %r10, %rbx
+; SSE-NEXT: adcq $0, %r11
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: mulq %r14
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: addq %rbx, %rax
+; SSE-NEXT: adcq %r11, %rcx
+; SSE-NEXT: setb %al
+; SSE-NEXT: movzbl %al, %r10d
+; SSE-NEXT: movq %r8, %rax
+; SSE-NEXT: mulq %r14
+; SSE-NEXT: addq %rcx, %rax
+; SSE-NEXT: adcq %r10, %rdx
+; SSE-NEXT: movq %rax, 16(%rdi)
+; SSE-NEXT: movq %r9, (%rdi)
+; SSE-NEXT: movq %rdx, 24(%rdi)
+; SSE-NEXT: movq %rsi, 8(%rdi)
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: umulh_v2i128:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdx, %rax
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %r9, %r10, %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: mulxq %r9, %r9, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX2-NEXT: addq %r10, %r9
+; AVX2-NEXT: adcq $0, %rbx
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: mulxq %r14, %rdx, %r10
+; AVX2-NEXT: addq %r9, %rdx
+; AVX2-NEXT: adcq %rbx, %r10
+; AVX2-NEXT: setb %dl
+; AVX2-NEXT: movzbl %dl, %ebx
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: mulxq %r14, %r9, %rsi
+; AVX2-NEXT: addq %r10, %r9
+; AVX2-NEXT: adcq %rbx, %rsi
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: mulxq %r11, %rax, %rax
+; AVX2-NEXT: movq %r8, %rdx
+; AVX2-NEXT: mulxq %r11, %r10, %r11
+; AVX2-NEXT: addq %rax, %r10
+; AVX2-NEXT: adcq $0, %r11
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: mulxq %rbx, %rcx, %r14
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: addq %r10, %rcx
+; AVX2-NEXT: adcq %r11, %r14
+; AVX2-NEXT: setb %cl
+; AVX2-NEXT: movq %r8, %rdx
+; AVX2-NEXT: mulxq %rbx, %rdx, %rdi
+; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: addq %r14, %rdx
+; AVX2-NEXT: adcq %rcx, %rdi
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r9, (%rax)
+; AVX2-NEXT: movq %rdi, 24(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: umulh_v2i128:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdx, %r10
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %r9, %r14, %r14
+; AVX512-NEXT: movq %r10, %rdx
+; AVX512-NEXT: mulxq %r9, %r9, %r15
+; AVX512-NEXT: addq %r14, %r9
+; AVX512-NEXT: adcq $0, %r15
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: mulxq %rbx, %rdx, %r14
+; AVX512-NEXT: addq %r9, %rdx
+; AVX512-NEXT: adcq %r15, %r14
+; AVX512-NEXT: setb %dl
+; AVX512-NEXT: movzbl %dl, %r9d
+; AVX512-NEXT: movq %r10, %rdx
+; AVX512-NEXT: mulxq %rbx, %r10, %rsi
+; AVX512-NEXT: addq %r14, %r10
+; AVX512-NEXT: adcq %r9, %rsi
+; AVX512-NEXT: movq %rcx, %rdx
+; AVX512-NEXT: mulxq %r11, %r9, %r9
+; AVX512-NEXT: movq %r8, %rdx
+; AVX512-NEXT: mulxq %r11, %r11, %rbx
+; AVX512-NEXT: addq %r9, %r11
+; AVX512-NEXT: adcq $0, %rbx
+; AVX512-NEXT: movq %rcx, %rdx
+; AVX512-NEXT: mulxq %rdi, %rcx, %r9
+; AVX512-NEXT: addq %r11, %rcx
+; AVX512-NEXT: adcq %rbx, %r9
+; AVX512-NEXT: setb %cl
+; AVX512-NEXT: movzbl %cl, %ecx
+; AVX512-NEXT: movq %r8, %rdx
+; AVX512-NEXT: mulxq %rdi, %rdx, %rdi
+; AVX512-NEXT: addq %r9, %rdx
+; AVX512-NEXT: adcq %rcx, %rdi
+; AVX512-NEXT: movq %rdx, 16(%rax)
+; AVX512-NEXT: movq %r10, (%rax)
+; AVX512-NEXT: movq %rdi, 24(%rax)
+; AVX512-NEXT: movq %rsi, 8(%rax)
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: retq
+ %r = call <2 x i128> @llvm.umulh(<2 x i128> %x, <2 x i128> %y)
+ ret <2 x i128> %r
+}
More information about the llvm-commits
mailing list