[llvm] 0b88e33 - [LLVM][CodeGen][SVE] Add custom lowering for ISD::MASKED_SDIV/UDIV. (#191164)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 05:42:25 PDT 2026
Author: Paul Walker
Date: 2026-04-17T13:42:19+01:00
New Revision: 0b88e333b0fee5d78bbc66f62369230c1b29ff39
URL: https://github.com/llvm/llvm-project/commit/0b88e333b0fee5d78bbc66f62369230c1b29ff39
DIFF: https://github.com/llvm/llvm-project/commit/0b88e333b0fee5d78bbc66f62369230c1b29ff39.diff
LOG: [LLVM][CodeGen][SVE] Add custom lowering for ISD::MASKED_SDIV/UDIV. (#191164)
Also refactor custom lowering of ISD::SDIV/UDIV to replace uses of
Arch64ISD::PRED_SDIV/UDIV with the new target independent equivalents.
Added:
llvm/test/CodeGen/AArch64/sve-fixed-length-masked-div.ll
llvm/test/CodeGen/AArch64/sve-fixed-length-masked-rem.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
llvm/test/Analysis/CostModel/AArch64/masked-divrem.ll
llvm/test/CodeGen/AArch64/masked-sdiv-fixed-length.ll
llvm/test/CodeGen/AArch64/masked-sdiv-scalable.ll
llvm/test/CodeGen/AArch64/masked-udiv-fixed-length.ll
llvm/test/CodeGen/AArch64/masked-udiv-scalable.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9b34d9b385b4e..e926ef0c97baf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1671,6 +1671,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::OR, VT, Custom);
}
+ for (auto VT : {MVT::nxv4i32, MVT::nxv2i64}) {
+ setOperationAction(ISD::MASKED_UDIV, VT, Legal);
+ setOperationAction(ISD::MASKED_SDIV, VT, Legal);
+ }
+
// Illegal unpacked integer vector types.
for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
@@ -1917,6 +1922,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// NEON but are available in SVE.
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ setOperationAction(ISD::MASKED_SDIV, VT, Custom);
+ setOperationAction(ISD::MASKED_UDIV, VT, Custom);
setOperationAction(ISD::SDIV, VT, Custom);
setOperationAction(ISD::UDIV, VT, Custom);
setOperationAction(ISD::MULHS, VT, Custom);
@@ -2504,6 +2511,8 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Default);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Default);
setOperationAction(ISD::LOAD, VT, PreferNEON ? Legal : Default);
+ setOperationAction(ISD::MASKED_UDIV, VT, Default);
+ setOperationAction(ISD::MASKED_SDIV, VT, Default);
setOperationAction(ISD::MGATHER, VT, PreferSVE ? Default : Expand);
setOperationAction(ISD::MLOAD, VT, Default);
setOperationAction(ISD::MSCATTER, VT, PreferSVE ? Default : Expand);
@@ -8191,6 +8200,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerEXTRACT_SUBVECTOR(Op, DAG);
case ISD::INSERT_SUBVECTOR:
return LowerINSERT_SUBVECTOR(Op, DAG);
+ case ISD::MASKED_SDIV:
+ case ISD::MASKED_UDIV:
case ISD::SDIV:
case ISD::UDIV:
return LowerDIV(Op, DAG);
@@ -16780,10 +16791,10 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
+ unsigned Opc = Op.getOpcode();
+ assert((Opc == ISD::SDIV || Opc == ISD::UDIV) && "Expected a DIV opcode.");
assert(VT.isScalableVector() && "Expected a scalable vector.");
-
- bool Signed = Op.getOpcode() == ISD::SDIV;
- unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
+ bool Signed = Opc == ISD::SDIV;
bool Negated;
uint64_t SplatVal;
@@ -16800,8 +16811,11 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
return Res;
}
- if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64)
- return LowerToPredicatedOp(Op, DAG, PredOpcode);
+ if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64) {
+ unsigned MaskedOpcode = Signed ? ISD::MASKED_SDIV : ISD::MASKED_UDIV;
+ return DAG.getNode(MaskedOpcode, DL, VT, Op.getOperand(0), Op.getOperand(1),
+ getPredicateForVector(DAG, DL, VT));
+ }
// SVE doesn't have i8 and i16 DIV operations; widen them to 32-bit
// operations, and truncate the result.
@@ -16819,8 +16833,8 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
SDValue Op1Lo = DAG.getNode(UnpkLo, DL, WidenedVT, Op.getOperand(1));
SDValue Op0Hi = DAG.getNode(UnpkHi, DL, WidenedVT, Op.getOperand(0));
SDValue Op1Hi = DAG.getNode(UnpkHi, DL, WidenedVT, Op.getOperand(1));
- SDValue ResultLo = DAG.getNode(Op.getOpcode(), DL, WidenedVT, Op0Lo, Op1Lo);
- SDValue ResultHi = DAG.getNode(Op.getOpcode(), DL, WidenedVT, Op0Hi, Op1Hi);
+ SDValue ResultLo = DAG.getNode(Opc, DL, WidenedVT, Op0Lo, Op1Lo);
+ SDValue ResultHi = DAG.getNode(Opc, DL, WidenedVT, Op0Hi, Op1Hi);
SDValue ResultLoCast = DAG.getNode(AArch64ISD::NVCAST, DL, VT, ResultLo);
SDValue ResultHiCast = DAG.getNode(AArch64ISD::NVCAST, DL, VT, ResultHi);
return DAG.getNode(AArch64ISD::UZP1, DL, VT, ResultLoCast, ResultHiCast);
@@ -23974,12 +23988,6 @@ static SDValue performIntrinsicCombine(SDNode *N,
return V;
return DAG.getNode(AArch64ISD::ABDU_PRED, SDLoc(N), N->getValueType(0),
N->getOperand(1), N->getOperand(2), N->getOperand(3));
- case Intrinsic::aarch64_sve_sdiv_u:
- return DAG.getNode(AArch64ISD::SDIV_PRED, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2), N->getOperand(3));
- case Intrinsic::aarch64_sve_udiv_u:
- return DAG.getNode(AArch64ISD::UDIV_PRED, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_sqadd:
return convertMergedOpToPredOp(N, ISD::SADDSAT, DAG, true);
case Intrinsic::aarch64_sve_sqsub_u:
@@ -31514,9 +31522,9 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
SDLoc DL(Op);
EVT VT = Op.getValueType();
EVT EltVT = VT.getVectorElementType();
-
- bool Signed = Op.getOpcode() == ISD::SDIV;
- unsigned PredOpcode = Signed ? AArch64ISD::SDIV_PRED : AArch64ISD::UDIV_PRED;
+ unsigned Opc = Op.getOpcode();
+ bool Signed = Opc == ISD::SDIV || Opc == ISD::MASKED_SDIV;
+ bool Masked = Opc == ISD::MASKED_SDIV || Opc == ISD::MASKED_UDIV;
bool Negated;
uint64_t SplatVal;
@@ -31526,8 +31534,10 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), DL, MVT::i32);
+ SDValue Pg = Masked
+ ? convertFixedMaskToScalableVector(Op.getOperand(2), DAG)
+ : getPredicateForFixedLengthVector(DAG, DL, VT);
- SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
SDValue Res =
DAG.getNode(AArch64ISD::ASRD_MERGE_OP1, DL, ContainerVT, Pg, Op1, Op2);
if (Negated)
@@ -31538,8 +31548,23 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
}
// Scalable vector i32/i64 DIV is supported.
- if (EltVT == MVT::i32 || EltVT == MVT::i64)
- return LowerToPredicatedOp(Op, DAG, PredOpcode);
+ if (EltVT == MVT::i32 || EltVT == MVT::i64) {
+ EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+ SDValue LHS = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
+ SDValue RHS = convertToScalableVector(DAG, ContainerVT, Op.getOperand(1));
+ SDValue Mask = Masked
+ ? convertFixedMaskToScalableVector(Op.getOperand(2), DAG)
+ : getPredicateForFixedLengthVector(DAG, DL, VT);
+
+ unsigned MaskedOpcode = Signed ? ISD::MASKED_SDIV : ISD::MASKED_UDIV;
+ SDValue Div = DAG.getNode(MaskedOpcode, DL, ContainerVT, LHS, RHS, Mask);
+ return convertFromScalableVector(DAG, VT, Div);
+ }
+
+ // Custom lower requires splitting the mask, which increases complexity with
+ // no real improvement to generated code compared to default expansion.
+ if (Masked)
+ return SDValue();
// Scalable vector i8/i16 DIV is not supported. Promote it to i32.
EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
@@ -31551,7 +31576,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
if (DAG.getTargetLoweringInfo().isTypeLegal(WideVT)) {
SDValue Op0 = DAG.getNode(ExtendOpcode, DL, WideVT, Op.getOperand(0));
SDValue Op1 = DAG.getNode(ExtendOpcode, DL, WideVT, Op.getOperand(1));
- SDValue Div = DAG.getNode(Op.getOpcode(), DL, WideVT, Op0, Op1);
+ SDValue Div = DAG.getNode(Opc, DL, WideVT, Op0, Op1);
return DAG.getNode(ISD::TRUNCATE, DL, VT, Div);
}
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 436f50fe451be..3dbe9e8ceb68d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -233,14 +233,12 @@ def AArch64mul_p : SDNode<"AArch64ISD::MUL_PRED", SDT_AArch64Arith>;
def AArch64sabd_p : SDNode<"AArch64ISD::ABDS_PRED", SDT_AArch64Arith>;
def AArch64shadd_p : SDNode<"AArch64ISD::HADDS_PRED", SDT_AArch64Arith>;
def AArch64srhadd_p : SDNode<"AArch64ISD::RHADDS_PRED", SDT_AArch64Arith>;
-def AArch64sdiv_p : SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
def AArch64smax_p : SDNode<"AArch64ISD::SMAX_PRED", SDT_AArch64Arith>;
def AArch64smin_p : SDNode<"AArch64ISD::SMIN_PRED", SDT_AArch64Arith>;
def AArch64smulh_p : SDNode<"AArch64ISD::MULHS_PRED", SDT_AArch64Arith>;
def AArch64uabd_p : SDNode<"AArch64ISD::ABDU_PRED", SDT_AArch64Arith>;
def AArch64uhadd_p : SDNode<"AArch64ISD::HADDU_PRED", SDT_AArch64Arith>;
def AArch64urhadd_p : SDNode<"AArch64ISD::RHADDU_PRED", SDT_AArch64Arith>;
-def AArch64udiv_p : SDNode<"AArch64ISD::UDIV_PRED", SDT_AArch64Arith>;
def AArch64umax_p : SDNode<"AArch64ISD::UMAX_PRED", SDT_AArch64Arith>;
def AArch64umin_p : SDNode<"AArch64ISD::UMIN_PRED", SDT_AArch64Arith>;
def AArch64umulh_p : SDNode<"AArch64ISD::MULHU_PRED", SDT_AArch64Arith>;
@@ -264,6 +262,13 @@ def AArch64fmaxnm_p_nnan : PatFrag<(ops node:$op1, node:$op2, node:$op3),
return N->getFlags().hasNoNaNs();
}]>;
+def AArch64sdiv_p : PatFrags<(ops node:$op1, node:$op2, node:$op3),
+ [(int_aarch64_sve_sdiv_u node:$op1, node:$op2, node:$op3),
+ (masked_sdiv node:$op2, node:$op3, node:$op1)]>;
+def AArch64udiv_p : PatFrags<(ops node:$op1, node:$op2, node:$op3),
+ [(int_aarch64_sve_udiv_u node:$op1, node:$op2, node:$op3),
+ (masked_udiv node:$op2, node:$op3, node:$op1)]>;
+
def SDT_AArch64Arith_Imm : SDTypeProfile<1, 3, [
SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVT<3,i32>,
SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>, SDTCisSameAs<0,2>
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked-divrem.ll b/llvm/test/Analysis/CostModel/AArch64/masked-divrem.ll
index 28013662db0e9..f93f20fceabce 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked-divrem.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked-divrem.ll
@@ -39,36 +39,36 @@ define void @udiv() {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE-LABEL: 'udiv'
-; SVE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:6 Lat:7 SizeLat:7 for: %V1I16 = call <1 x i16> @llvm.masked.udiv.v1i16(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:8 Lat:10 SizeLat:10 for: %V2I16 = call <2 x i16> @llvm.masked.udiv.v2i16(<2 x i16> poison, <2 x i16> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:12 Lat:16 SizeLat:16 for: %V4I16 = call <4 x i16> @llvm.masked.udiv.v4i16(<4 x i16> poison, <4 x i16> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:20 Lat:28 SizeLat:28 for: %V8I16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:20 Lat:20 SizeLat:20 for: %V16I16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:128 CodeSize:68 Lat:100 SizeLat:100 for: %V32I16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V1I16 = call <1 x i16> @llvm.masked.udiv.v1i16(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2I16 = call <2 x i16> @llvm.masked.udiv.v2i16(<2 x i16> poison, <2 x i16> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4I16 = call <4 x i16> @llvm.masked.udiv.v4i16(<4 x i16> poison, <4 x i16> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8I16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32I16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I16 = call <vscale x 1 x i16> @llvm.masked.udiv.nxv1i16(<vscale x 1 x i16> poison, <vscale x 1 x i16> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I16 = call <vscale x 2 x i16> @llvm.masked.udiv.nxv2i16(<vscale x 2 x i16> poison, <vscale x 2 x i16> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV4I16 = call <vscale x 4 x i16> @llvm.masked.udiv.nxv4i16(<vscale x 4 x i16> poison, <vscale x 4 x i16> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I16 = call <vscale x 2 x i16> @llvm.masked.udiv.nxv2i16(<vscale x 2 x i16> poison, <vscale x 2 x i16> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV4I16 = call <vscale x 4 x i16> @llvm.masked.udiv.nxv4i16(<vscale x 4 x i16> poison, <vscale x 4 x i16> poison, <vscale x 4 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %NXV8I16 = call <vscale x 8 x i16> @llvm.masked.udiv.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison, <vscale x 8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:18 CodeSize:6 Lat:6 SizeLat:6 for: %NXV16I16 = call <vscale x 16 x i16> @llvm.masked.udiv.nxv16i16(<vscale x 16 x i16> poison, <vscale x 16 x i16> poison, <vscale x 16 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:36 CodeSize:8 Lat:8 SizeLat:8 for: %NXV32I16 = call <vscale x 32 x i16> @llvm.masked.udiv.nxv32i16(<vscale x 32 x i16> poison, <vscale x 32 x i16> poison, <vscale x 32 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:7 SizeLat:7 for: %V1I32 = call <1 x i32> @llvm.masked.udiv.v1i32(<1 x i32> poison, <1 x i32> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:10 SizeLat:10 for: %V2I32 = call <2 x i32> @llvm.masked.udiv.v2i32(<2 x i32> poison, <2 x i32> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:14 CodeSize:12 Lat:16 SizeLat:16 for: %V4I32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 12 for: %V8I32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:24 CodeSize:20 Lat:20 SizeLat:20 for: %V16I32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V1I32 = call <1 x i32> @llvm.masked.udiv.v1i32(<1 x i32> poison, <1 x i32> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %V2I32 = call <2 x i32> @llvm.masked.udiv.v2i32(<2 x i32> poison, <2 x i32> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4I32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %V8I32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16I32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I32 = call <vscale x 1 x i32> @llvm.masked.udiv.nxv1i32(<vscale x 1 x i32> poison, <vscale x 1 x i32> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I32 = call <vscale x 2 x i32> @llvm.masked.udiv.nxv2i32(<vscale x 2 x i32> poison, <vscale x 2 x i32> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV4I32 = call <vscale x 4 x i32> @llvm.masked.udiv.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison, <vscale x 4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 6 for: %NXV8I32 = call <vscale x 8 x i32> @llvm.masked.udiv.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, <vscale x 8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %NXV16I32 = call <vscale x 16 x i32> @llvm.masked.udiv.nxv16i32(<vscale x 16 x i32> poison, <vscale x 16 x i32> poison, <vscale x 16 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:7 SizeLat:7 for: %V1I64 = call <1 x i64> @llvm.masked.udiv.v1i64(<1 x i64> poison, <1 x i64> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:10 for: %V2I64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 84 for: %V4I64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:168 CodeSize:164 Lat:164 SizeLat:164 for: %V8I64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I32 = call <vscale x 2 x i32> @llvm.masked.udiv.nxv2i32(<vscale x 2 x i32> poison, <vscale x 2 x i32> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV4I32 = call <vscale x 4 x i32> @llvm.masked.udiv.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %NXV8I32 = call <vscale x 8 x i32> @llvm.masked.udiv.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, <vscale x 8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NXV16I32 = call <vscale x 16 x i32> @llvm.masked.udiv.nxv16i32(<vscale x 16 x i32> poison, <vscale x 16 x i32> poison, <vscale x 16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V1I64 = call <1 x i64> @llvm.masked.udiv.v1i64(<1 x i64> poison, <1 x i64> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V2I64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %V4I64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8I64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I64 = call <vscale x 1 x i64> @llvm.masked.udiv.nxv1i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I64 = call <vscale x 2 x i64> @llvm.masked.udiv.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 6 for: %NXV4I64 = call <vscale x 4 x i64> @llvm.masked.udiv.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison, <vscale x 4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %NXV8I64 = call <vscale x 8 x i64> @llvm.masked.udiv.nxv8i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> poison, <vscale x 8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I64 = call <vscale x 2 x i64> @llvm.masked.udiv.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %NXV4I64 = call <vscale x 4 x i64> @llvm.masked.udiv.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NXV8I64 = call <vscale x 8 x i64> @llvm.masked.udiv.nxv8i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> poison, <vscale x 8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V1I16 = call <1 x i16> @llvm.masked.udiv(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
@@ -145,36 +145,36 @@ define void @sdiv() {
; NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE-LABEL: 'sdiv'
-; SVE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:6 Lat:7 SizeLat:7 for: %V1I16 = call <1 x i16> @llvm.masked.sdiv.v1i16(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:11 CodeSize:8 Lat:10 SizeLat:10 for: %V2I16 = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> poison, <2 x i16> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:12 Lat:16 SizeLat:16 for: %V4I16 = call <4 x i16> @llvm.masked.sdiv.v4i16(<4 x i16> poison, <4 x i16> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:20 Lat:28 SizeLat:28 for: %V8I16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:20 Lat:20 SizeLat:20 for: %V16I16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:128 CodeSize:68 Lat:100 SizeLat:100 for: %V32I16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V1I16 = call <1 x i16> @llvm.masked.sdiv.v1i16(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2I16 = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> poison, <2 x i16> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4I16 = call <4 x i16> @llvm.masked.sdiv.v4i16(<4 x i16> poison, <4 x i16> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8I16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16I16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32I16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I16 = call <vscale x 1 x i16> @llvm.masked.sdiv.nxv1i16(<vscale x 1 x i16> poison, <vscale x 1 x i16> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I16 = call <vscale x 2 x i16> @llvm.masked.sdiv.nxv2i16(<vscale x 2 x i16> poison, <vscale x 2 x i16> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV4I16 = call <vscale x 4 x i16> @llvm.masked.sdiv.nxv4i16(<vscale x 4 x i16> poison, <vscale x 4 x i16> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I16 = call <vscale x 2 x i16> @llvm.masked.sdiv.nxv2i16(<vscale x 2 x i16> poison, <vscale x 2 x i16> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV4I16 = call <vscale x 4 x i16> @llvm.masked.sdiv.nxv4i16(<vscale x 4 x i16> poison, <vscale x 4 x i16> poison, <vscale x 4 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %NXV8I16 = call <vscale x 8 x i16> @llvm.masked.sdiv.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison, <vscale x 8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:18 CodeSize:6 Lat:6 SizeLat:6 for: %NXV16I16 = call <vscale x 16 x i16> @llvm.masked.sdiv.nxv16i16(<vscale x 16 x i16> poison, <vscale x 16 x i16> poison, <vscale x 16 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:36 CodeSize:8 Lat:8 SizeLat:8 for: %NXV32I16 = call <vscale x 32 x i16> @llvm.masked.sdiv.nxv32i16(<vscale x 32 x i16> poison, <vscale x 32 x i16> poison, <vscale x 32 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:7 SizeLat:7 for: %V1I32 = call <1 x i32> @llvm.masked.sdiv.v1i32(<1 x i32> poison, <1 x i32> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:7 CodeSize:8 Lat:10 SizeLat:10 for: %V2I32 = call <2 x i32> @llvm.masked.sdiv.v2i32(<2 x i32> poison, <2 x i32> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:14 CodeSize:12 Lat:16 SizeLat:16 for: %V4I32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 12 for: %V8I32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:24 CodeSize:20 Lat:20 SizeLat:20 for: %V16I32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V1I32 = call <1 x i32> @llvm.masked.sdiv.v1i32(<1 x i32> poison, <1 x i32> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %V2I32 = call <2 x i32> @llvm.masked.sdiv.v2i32(<2 x i32> poison, <2 x i32> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4I32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %V8I32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16I32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I32 = call <vscale x 1 x i32> @llvm.masked.sdiv.nxv1i32(<vscale x 1 x i32> poison, <vscale x 1 x i32> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I32 = call <vscale x 2 x i32> @llvm.masked.sdiv.nxv2i32(<vscale x 2 x i32> poison, <vscale x 2 x i32> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV4I32 = call <vscale x 4 x i32> @llvm.masked.sdiv.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison, <vscale x 4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 6 for: %NXV8I32 = call <vscale x 8 x i32> @llvm.masked.sdiv.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, <vscale x 8 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %NXV16I32 = call <vscale x 16 x i32> @llvm.masked.sdiv.nxv16i32(<vscale x 16 x i32> poison, <vscale x 16 x i32> poison, <vscale x 16 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:6 Lat:7 SizeLat:7 for: %V1I64 = call <1 x i64> @llvm.masked.sdiv.v1i64(<1 x i64> poison, <1 x i64> poison, <1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:10 for: %V2I64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 84 for: %V4I64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:168 CodeSize:164 Lat:164 SizeLat:164 for: %V8I64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I32 = call <vscale x 2 x i32> @llvm.masked.sdiv.nxv2i32(<vscale x 2 x i32> poison, <vscale x 2 x i32> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV4I32 = call <vscale x 4 x i32> @llvm.masked.sdiv.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %NXV8I32 = call <vscale x 8 x i32> @llvm.masked.sdiv.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, <vscale x 8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NXV16I32 = call <vscale x 16 x i32> @llvm.masked.sdiv.nxv16i32(<vscale x 16 x i32> poison, <vscale x 16 x i32> poison, <vscale x 16 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V1I64 = call <1 x i64> @llvm.masked.sdiv.v1i64(<1 x i64> poison, <1 x i64> poison, <1 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V2I64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %V4I64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8I64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of Invalid for: %NXV1I64 = call <vscale x 1 x i64> @llvm.masked.sdiv.nxv1i64(<vscale x 1 x i64> poison, <vscale x 1 x i64> poison, <vscale x 1 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:5 SizeLat:5 for: %NXV2I64 = call <vscale x 2 x i64> @llvm.masked.sdiv.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison, <vscale x 2 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of 6 for: %NXV4I64 = call <vscale x 4 x i64> @llvm.masked.sdiv.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison, <vscale x 4 x i1> poison)
-; SVE-NEXT: Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %NXV8I64 = call <vscale x 8 x i64> @llvm.masked.sdiv.nxv8i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> poison, <vscale x 8 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %NXV2I64 = call <vscale x 2 x i64> @llvm.masked.sdiv.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison, <vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of 4 for: %NXV4I64 = call <vscale x 4 x i64> @llvm.masked.sdiv.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison, <vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %NXV8I64 = call <vscale x 8 x i64> @llvm.masked.sdiv.nxv8i64(<vscale x 8 x i64> poison, <vscale x 8 x i64> poison, <vscale x 8 x i1> poison)
; SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V1I16 = call <1 x i16> @llvm.masked.sdiv(<1 x i16> poison, <1 x i16> poison, <1 x i1> poison)
diff --git a/llvm/test/CodeGen/AArch64/masked-sdiv-fixed-length.ll b/llvm/test/CodeGen/AArch64/masked-sdiv-fixed-length.ll
index 71740bbb2f4e5..0c0da0bc24113 100644
--- a/llvm/test/CodeGen/AArch64/masked-sdiv-fixed-length.ll
+++ b/llvm/test/CodeGen/AArch64/masked-sdiv-fixed-length.ll
@@ -35,12 +35,10 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: ptrue p0.s, vl4
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: shl v2.4s, v2.4s, #31
-; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: and v1.16b, v1.16b, v2.16b
-; SVE-NEXT: mvn v2.16b, v2.16b
-; SVE-NEXT: sub v1.4s, v1.4s, v2.4s
-; SVE-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; SVE-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <4 x i32> @llvm.masked.sdiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
@@ -71,12 +69,10 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
; SVE-NEXT: ushll v2.2d, v2.2s, #0
; SVE-NEXT: ptrue p0.d, vl2
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: shl v2.2d, v2.2d, #63
-; SVE-NEXT: cmlt v2.2d, v2.2d, #0
-; SVE-NEXT: and v1.16b, v1.16b, v2.16b
-; SVE-NEXT: mvn v2.16b, v2.16b
-; SVE-NEXT: sub v1.2d, v1.2d, v2.2d
-; SVE-NEXT: sdiv z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; SVE-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <2 x i64> @llvm.masked.sdiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
@@ -122,23 +118,19 @@ define <4 x i64> @sdiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
; SVE: // %bb.0:
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: ushll2 v5.2d, v4.4s, #0
; SVE-NEXT: shl v5.2d, v5.2d, #63
-; SVE-NEXT: cmlt v5.2d, v5.2d, #0
-; SVE-NEXT: and v3.16b, v3.16b, v5.16b
-; SVE-NEXT: mvn v5.16b, v5.16b
-; SVE-NEXT: sub v3.2d, v3.2d, v5.2d
-; SVE-NEXT: sdiv z1.d, p0/m, z1.d, z3.d
+; SVE-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; SVE-NEXT: sdiv z1.d, p1/m, z1.d, z3.d
; SVE-NEXT: ushll v3.2d, v4.2s, #0
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: shl v3.2d, v3.2d, #63
-; SVE-NEXT: cmlt v3.2d, v3.2d, #0
-; SVE-NEXT: and v2.16b, v2.16b, v3.16b
-; SVE-NEXT: mvn v3.16b, v3.16b
-; SVE-NEXT: sub v2.2d, v2.2d, v3.2d
-; SVE-NEXT: sdiv z0.d, p0/m, z0.d, z2.d
+; SVE-NEXT: cmpne p1.d, p0/z, z3.d, #0
+; SVE-NEXT: sdiv z0.d, p1/m, z0.d, z2.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
@@ -171,11 +163,9 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SVE-NEXT: shl v2.2s, v2.2s, #31
; SVE-NEXT: ptrue p0.s, vl2
; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: cmlt v2.2s, v2.2s, #0
-; SVE-NEXT: and v1.8b, v1.8b, v2.8b
-; SVE-NEXT: mvn v2.8b, v2.8b
-; SVE-NEXT: sub v1.2s, v1.2s, v2.2s
-; SVE-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: // kill: def $d1 killed $d1 def $z1
+; SVE-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; SVE-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%res = call <2 x i32> @llvm.masked.sdiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
diff --git a/llvm/test/CodeGen/AArch64/masked-sdiv-scalable.ll b/llvm/test/CodeGen/AArch64/masked-sdiv-scalable.ll
index bb3ce4124907a..6cd2575f75a8e 100644
--- a/llvm/test/CodeGen/AArch64/masked-sdiv-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/masked-sdiv-scalable.ll
@@ -5,11 +5,9 @@ define <vscale x 4 x i16> @sdiv_nxv4i16(<vscale x 4 x i16> %x, <vscale x 4 x i16
; CHECK-LABEL: sdiv_nxv4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: mov z2.s, #1 // =0x1
; CHECK-NEXT: sxth z1.s, p1/m, z1.s
; CHECK-NEXT: sxth z0.s, p1/m, z0.s
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i16> @llvm.masked.sdiv(<vscale x 4 x i16> %x, <vscale x 4 x i16> %y, <vscale x 4 x i1> %m)
ret <vscale x 4 x i16> %res
@@ -18,9 +16,6 @@ define <vscale x 4 x i16> @sdiv_nxv4i16(<vscale x 4 x i16> %x, <vscale x 4 x i16
define <vscale x 4 x i32> @sdiv_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y, <vscale x 4 x i1> %m) {
; CHECK-LABEL: sdiv_nxv4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.s, #1 // =0x1
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i32> @llvm.masked.sdiv(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y, <vscale x 4 x i1> %m)
@@ -30,14 +25,10 @@ define <vscale x 4 x i32> @sdiv_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32
define <vscale x 8 x i32> @sdiv_nxv8i32(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y, <vscale x 8 x i1> %m) {
; CHECK-LABEL: sdiv_nxv8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z4.s, #1 // =0x1
; CHECK-NEXT: punpklo p1.h, p0.b
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: sel z2.s, p1, z2.s, z4.s
-; CHECK-NEXT: ptrue p1.s
; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z2.s
-; CHECK-NEXT: sel z2.s, p0, z3.s, z4.s
-; CHECK-NEXT: sdiv z1.s, p1/m, z1.s, z2.s
+; CHECK-NEXT: sdiv z1.s, p0/m, z1.s, z3.s
; CHECK-NEXT: ret
%res = call <vscale x 8 x i32> @llvm.masked.sdiv(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y, <vscale x 8 x i1> %m)
ret <vscale x 8 x i32> %res
@@ -64,9 +55,6 @@ define <vscale x 8 x i16> @sdiv_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16
define <vscale x 2 x i64> @sdiv_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y, <vscale x 2 x i1> %m) {
; CHECK-LABEL: sdiv_nxv2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, #1 // =0x1
-; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%res = call <vscale x 2 x i64> @llvm.masked.sdiv(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y, <vscale x 2 x i1> %m)
diff --git a/llvm/test/CodeGen/AArch64/masked-udiv-fixed-length.ll b/llvm/test/CodeGen/AArch64/masked-udiv-fixed-length.ll
index d21591ed90127..3e53acabf81ee 100644
--- a/llvm/test/CodeGen/AArch64/masked-udiv-fixed-length.ll
+++ b/llvm/test/CodeGen/AArch64/masked-udiv-fixed-length.ll
@@ -35,12 +35,10 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SVE-NEXT: ushll v2.4s, v2.4h, #0
; SVE-NEXT: ptrue p0.s, vl4
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: shl v2.4s, v2.4s, #31
-; SVE-NEXT: cmlt v2.4s, v2.4s, #0
-; SVE-NEXT: and v1.16b, v1.16b, v2.16b
-; SVE-NEXT: mvn v2.16b, v2.16b
-; SVE-NEXT: sub v1.4s, v1.4s, v2.4s
-; SVE-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; SVE-NEXT: udiv z0.s, p1/m, z0.s, z1.s
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <4 x i32> @llvm.masked.udiv(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m)
@@ -71,12 +69,10 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) {
; SVE-NEXT: ushll v2.2d, v2.2s, #0
; SVE-NEXT: ptrue p0.d, vl2
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: shl v2.2d, v2.2d, #63
-; SVE-NEXT: cmlt v2.2d, v2.2d, #0
-; SVE-NEXT: and v1.16b, v1.16b, v2.16b
-; SVE-NEXT: mvn v2.16b, v2.16b
-; SVE-NEXT: sub v1.2d, v1.2d, v2.2d
-; SVE-NEXT: udiv z0.d, p0/m, z0.d, z1.d
+; SVE-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; SVE-NEXT: udiv z0.d, p1/m, z0.d, z1.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <2 x i64> @llvm.masked.udiv(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m)
@@ -122,23 +118,19 @@ define <4 x i64> @udiv_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m) {
; SVE: // %bb.0:
; SVE-NEXT: ushll v4.4s, v4.4h, #0
; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
; SVE-NEXT: ushll2 v5.2d, v4.4s, #0
; SVE-NEXT: shl v5.2d, v5.2d, #63
-; SVE-NEXT: cmlt v5.2d, v5.2d, #0
-; SVE-NEXT: and v3.16b, v3.16b, v5.16b
-; SVE-NEXT: mvn v5.16b, v5.16b
-; SVE-NEXT: sub v3.2d, v3.2d, v5.2d
-; SVE-NEXT: udiv z1.d, p0/m, z1.d, z3.d
+; SVE-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; SVE-NEXT: udiv z1.d, p1/m, z1.d, z3.d
; SVE-NEXT: ushll v3.2d, v4.2s, #0
; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: shl v3.2d, v3.2d, #63
-; SVE-NEXT: cmlt v3.2d, v3.2d, #0
-; SVE-NEXT: and v2.16b, v2.16b, v3.16b
-; SVE-NEXT: mvn v3.16b, v3.16b
-; SVE-NEXT: sub v2.2d, v2.2d, v3.2d
-; SVE-NEXT: udiv z0.d, p0/m, z0.d, z2.d
+; SVE-NEXT: cmpne p1.d, p0/z, z3.d, #0
+; SVE-NEXT: udiv z0.d, p1/m, z0.d, z2.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %x, <4 x i64> %y, <4 x i1> %m)
@@ -171,11 +163,9 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SVE-NEXT: shl v2.2s, v2.2s, #31
; SVE-NEXT: ptrue p0.s, vl2
; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: cmlt v2.2s, v2.2s, #0
-; SVE-NEXT: and v1.8b, v1.8b, v2.8b
-; SVE-NEXT: mvn v2.8b, v2.8b
-; SVE-NEXT: sub v1.2s, v1.2s, v2.2s
-; SVE-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: // kill: def $d1 killed $d1 def $z1
+; SVE-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; SVE-NEXT: udiv z0.s, p1/m, z0.s, z1.s
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%res = call <2 x i32> @llvm.masked.udiv(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m)
diff --git a/llvm/test/CodeGen/AArch64/masked-udiv-scalable.ll b/llvm/test/CodeGen/AArch64/masked-udiv-scalable.ll
index 254320de423f3..1e5b34c0e9078 100644
--- a/llvm/test/CodeGen/AArch64/masked-udiv-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/masked-udiv-scalable.ll
@@ -5,10 +5,7 @@ define <vscale x 4 x i16> @udiv_nxv4i16(<vscale x 4 x i16> %x, <vscale x 4 x i16
; CHECK-LABEL: udiv_nxv4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: and z1.s, z1.s, #0xffff
-; CHECK-NEXT: mov z2.s, #1 // =0x1
; CHECK-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i16> @llvm.masked.udiv(<vscale x 4 x i16> %x, <vscale x 4 x i16> %y, <vscale x 4 x i1> %m)
@@ -18,9 +15,6 @@ define <vscale x 4 x i16> @udiv_nxv4i16(<vscale x 4 x i16> %x, <vscale x 4 x i16
define <vscale x 4 x i32> @udiv_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y, <vscale x 4 x i1> %m) {
; CHECK-LABEL: udiv_nxv4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.s, #1 // =0x1
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i32> @llvm.masked.udiv(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y, <vscale x 4 x i1> %m)
@@ -30,14 +24,10 @@ define <vscale x 4 x i32> @udiv_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32
define <vscale x 8 x i32> @udiv_nxv8i32(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y, <vscale x 8 x i1> %m) {
; CHECK-LABEL: udiv_nxv8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z4.s, #1 // =0x1
; CHECK-NEXT: punpklo p1.h, p0.b
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: sel z2.s, p1, z2.s, z4.s
-; CHECK-NEXT: ptrue p1.s
; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z2.s
-; CHECK-NEXT: sel z2.s, p0, z3.s, z4.s
-; CHECK-NEXT: udiv z1.s, p1/m, z1.s, z2.s
+; CHECK-NEXT: udiv z1.s, p0/m, z1.s, z3.s
; CHECK-NEXT: ret
%res = call <vscale x 8 x i32> @llvm.masked.udiv(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y, <vscale x 8 x i1> %m)
ret <vscale x 8 x i32> %res
@@ -64,9 +54,6 @@ define <vscale x 8 x i16> @udiv_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16
define <vscale x 2 x i64> @udiv_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y, <vscale x 2 x i1> %m) {
; CHECK-LABEL: udiv_nxv2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, #1 // =0x1
-; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%res = call <vscale x 2 x i64> @llvm.masked.udiv(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y, <vscale x 2 x i1> %m)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-div.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-div.ll
new file mode 100644
index 0000000000000..4f782f4c7a6f9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-div.ll
@@ -0,0 +1,1693 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128
+; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; SDIV
+;
+
+; Vector vXi8 sdiv are not legal for NEON so use SVE when available.
+define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: sdiv_v8i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sshll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; VBITS_GE_128-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v8i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b
+; VBITS_GE_256-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v8i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b
+; VBITS_GE_512-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b
+; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i8> %op1, zeroinitializer
+ %res = call <8 x i8> @llvm.masked.sdiv(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask)
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: sdiv_v16i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sshll2 v2.8h, v0.16b, #0
+; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v3.8h, v1.16b, #0
+; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v1.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h
+; VBITS_GE_128-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v16i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: sunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h
+; VBITS_GE_256-NEXT: uzp1 z0.b, z1.b, z1.b
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v16i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b
+; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <16 x i8> %op1, zeroinitializer
+ %res = call <16 x i8> @llvm.masked.sdiv(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
+ ret <16 x i8> %res
+}
+
+define void @sdiv_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: sdiv_v32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: sunpklo z1.h, z1.b
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i8>, ptr %a
+ %op2 = load <32 x i8>, ptr %b
+ %mask = icmp ne <32 x i8> %op1, zeroinitializer
+ %res = call <32 x i8> @llvm.masked.sdiv(<32 x i8> %op1, <32 x i8> %op2, <32 x i1> %mask)
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl64
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: sunpklo z1.h, z1.b
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i8>, ptr %a
+ %op2 = load <64 x i8>, ptr %b
+ %mask = icmp ne <64 x i8> %op1, zeroinitializer
+ %res = call <64 x i8> @llvm.masked.sdiv(<64 x i8> %op1, <64 x i8> %op2, <64 x i1> %mask)
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v128i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: sunpklo z2.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: sunpklo z1.h, z1.b
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: sunpklo z3.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: st1b { z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i8>, ptr %a
+ %op2 = load <128 x i8>, ptr %b
+ %mask = icmp ne <128 x i8> %op1, zeroinitializer
+ %res = call <128 x i8> @llvm.masked.sdiv(<128 x i8> %op1, <128 x i8> %op2, <128 x i1> %mask)
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v256i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: sunpklo z2.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z3.h, z1.b
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: sunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: sunpklo z1.h, z1.b
+; CHECK-NEXT: sunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: sunpklo z2.s, z2.h
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: sdiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: sunpklo z5.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: sunpklo z3.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: sdiv z3.s, p1/m, z3.s, z5.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h
+; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT: ptrue p1.b, vl128
+; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b
+; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <256 x i8>, ptr %a
+ %op2 = load <256 x i8>, ptr %b
+ %mask = icmp ne <256 x i8> %op1, zeroinitializer
+ %res = call <256 x i8> @llvm.masked.sdiv(<256 x i8> %op1, <256 x i8> %op2, <256 x i1> %mask)
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+; Vector vXi16 sdiv are not legal for NEON so use SVE when available.
+define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
+; CHECK-LABEL: sdiv_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.4h, v1.4h, v2.4h
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i16> %op1, zeroinitializer
+ %res = call <4 x i16> @llvm.masked.sdiv(<4 x i16> %op1, <4 x i16> %op2, <4 x i1> %mask)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
+; VBITS_GE_128-LABEL: sdiv_v8i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_128-NEXT: sshll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v8i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v8i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i16> %op1, zeroinitializer
+ %res = call <8 x i16> @llvm.masked.sdiv(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask)
+ ret <8 x i16> %res
+}
+
+define void @sdiv_v16i16(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: sdiv_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldr q3, [x1, #16]
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: cmeq v5.8h, v1.8h, #0
+; VBITS_GE_128-NEXT: bic v3.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: ldr q4, [x1]
+; VBITS_GE_128-NEXT: bic v4.16b, v4.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v4.8h, v4.8h, v5.8h
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sshll2 v6.4s, v4.8h, #0
+; VBITS_GE_128-NEXT: sshll v4.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: sdiv z5.s, p0/m, z5.s, z6.s
+; VBITS_GE_128-NEXT: sdiv z1.s, p0/m, z1.s, z4.s
+; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z2.s
+; VBITS_GE_128-NEXT: uzp1 v1.8h, v1.8h, v5.8h
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov z1.h, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_256-NEXT: mov z1.h, p1/m, z2.h
+; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: sunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16
+; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ptrue p1.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v16i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl16
+; VBITS_GE_512-NEXT: mov z2.h, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: sel z1.h, p1, z1.h, z2.h
+; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: st1h { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i16>, ptr %a
+ %op2 = load <16 x i16>, ptr %b
+ %mask = icmp ne <16 x i16> %op1, zeroinitializer
+ %res = call <16 x i16> @llvm.masked.sdiv(<16 x i16> %op1, <16 x i16> %op2, <16 x i1> %mask)
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: sdiv_v32i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl32
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i16>, ptr %a
+ %op2 = load <32 x i16>, ptr %b
+ %mask = icmp ne <32 x i16> %op1, zeroinitializer
+ %res = call <32 x i16> @llvm.masked.sdiv(<32 x i16> %op1, <32 x i16> %op2, <32 x i1> %mask)
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v64i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i16>, ptr %a
+ %op2 = load <64 x i16>, ptr %b
+ %mask = icmp ne <64 x i16> %op1, zeroinitializer
+ %res = call <64 x i16> @llvm.masked.sdiv(<64 x i16> %op1, <64 x i16> %op2, <64 x i1> %mask)
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v128i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: mov z1.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: mov z1.h, p1/m, z2.h
+; CHECK-NEXT: sunpklo z2.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z3.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: sunpklo z1.s, z1.h
+; CHECK-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h
+; CHECK-NEXT: st1h { z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i16>, ptr %a
+ %op2 = load <128 x i16>, ptr %b
+ %mask = icmp ne <128 x i16> %op1, zeroinitializer
+ %res = call <128 x i16> @llvm.masked.sdiv(<128 x i16> %op1, <128 x i16> %op2, <128 x i1> %mask)
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+; Vector v2i32 sdiv are not legal for NEON so use SVE when available.
+define <2 x i32> @sdiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: sdiv_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i32> %op1, zeroinitializer
+ %res = call <2 x i32> @llvm.masked.sdiv(<2 x i32> %op1, <2 x i32> %op2, <2 x i1> %mask)
+ ret <2 x i32> %res
+}
+
+; Vector v4i32 sdiv are not legal for NEON so use SVE when available.
+define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: sdiv_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i32> %op1, zeroinitializer
+ %res = call <4 x i32> @llvm.masked.sdiv(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
+ ret <4 x i32> %res
+}
+
+define void @sdiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: sdiv_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <8 x i32>, ptr %a
+ %op2 = load <8 x i32>, ptr %b
+ %mask = icmp ne <8 x i32> %op1, zeroinitializer
+ %res = call <8 x i32> @llvm.masked.sdiv(<8 x i32> %op1, <8 x i32> %op2, <8 x i1> %mask)
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v16i32(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: sdiv_v16i32:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: cmeq v2.4s, v0.4s, #0
+; VBITS_GE_128-NEXT: cmeq v3.4s, v1.4s, #0
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: ldp q3, q4, [x0, #32]
+; VBITS_GE_128-NEXT: cmeq v5.4s, v4.4s, #0
+; VBITS_GE_128-NEXT: cmeq v6.4s, v3.4s, #0
+; VBITS_GE_128-NEXT: mvn v2.16b, v2.16b
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: zip1 v7.8b, v2.8b, v0.8b
+; VBITS_GE_128-NEXT: zip2 v2.8b, v2.8b, v0.8b
+; VBITS_GE_128-NEXT: xtn v5.8b, v5.8h
+; VBITS_GE_128-NEXT: ushll v6.4s, v7.4h, #0
+; VBITS_GE_128-NEXT: ldp q7, q16, [x1]
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v2.4s, v2.4s, #31
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z6.s, #0
+; VBITS_GE_128-NEXT: zip2 v6.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: zip1 v5.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: ushll v6.4s, v6.4h, #0
+; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: sdiv z1.s, p1/m, z1.s, z7.s
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v5.4s, v5.4s, #31
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z6.s, #0
+; VBITS_GE_128-NEXT: ldr q6, [x1, #48]
+; VBITS_GE_128-NEXT: sdiv z4.s, p1/m, z4.s, z6.s
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z5.s, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #32]
+; VBITS_GE_128-NEXT: sdiv z3.s, p1/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; VBITS_GE_128-NEXT: sdiv z0.s, p1/m, z0.s, z16.s
+; VBITS_GE_128-NEXT: stp q3, q4, [x0, #32]
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v16i32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_256-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; VBITS_GE_256-NEXT: sdiv z1.s, p1/m, z1.s, z2.s
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v16i32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_512-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i32>, ptr %a
+ %op2 = load <16 x i32>, ptr %b
+ %mask = icmp ne <16 x i32> %op1, zeroinitializer
+ %res = call <16 x i32> @llvm.masked.sdiv(<16 x i32> %op1, <16 x i32> %op2, <16 x i1> %mask)
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: sdiv_v32i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i32>, ptr %a
+ %op2 = load <32 x i32>, ptr %b
+ %mask = icmp ne <32 x i32> %op1, zeroinitializer
+ %res = call <32 x i32> @llvm.masked.sdiv(<32 x i32> %op1, <32 x i32> %op2, <32 x i1> %mask)
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v64i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i32>, ptr %a
+ %op2 = load <64 x i32>, ptr %b
+ %mask = icmp ne <64 x i32> %op1, zeroinitializer
+ %res = call <64 x i32> @llvm.masked.sdiv(<64 x i32> %op1, <64 x i32> %op2, <64 x i1> %mask)
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+; TODO: Vector i64 sdiv are not legal for NEON but the operation is scalarised
+; during type legalisation, so we don't get chance to use SVE.
+define <1 x i64> @sdiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: sdiv_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: fmov x9, d1
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csinc x9, x9, xzr, ne
+; CHECK-NEXT: sdiv x8, x8, x9
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+ %mask = icmp ne <1 x i64> %op1, zeroinitializer
+ %res = call <1 x i64> @llvm.masked.sdiv(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask)
+ ret <1 x i64> %res
+}
+
+; Vector i64 sdiv are not legal for NEON so use SVE when available.
+define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: sdiv_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i64> %op1, zeroinitializer
+ %res = call <2 x i64> @llvm.masked.sdiv(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
+ ret <2 x i64> %res
+}
+
+define void @sdiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: sdiv_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <4 x i64>, ptr %a
+ %op2 = load <4 x i64>, ptr %b
+ %mask = icmp ne <4 x i64> %op1, zeroinitializer
+ %res = call <4 x i64> @llvm.masked.sdiv(<4 x i64> %op1, <4 x i64> %op2, <4 x i1> %mask)
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v8i64(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: sdiv_v8i64:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q2, [x0, #32]
+; VBITS_GE_128-NEXT: ptrue p0.d, vl2
+; VBITS_GE_128-NEXT: ldp q3, q0, [x0]
+; VBITS_GE_128-NEXT: cmeq v4.2d, v2.2d, #0
+; VBITS_GE_128-NEXT: cmeq v5.2d, v1.2d, #0
+; VBITS_GE_128-NEXT: cmeq v6.2d, v0.2d, #0
+; VBITS_GE_128-NEXT: cmeq v7.2d, v3.2d, #0
+; VBITS_GE_128-NEXT: uzp1 v4.4s, v5.4s, v4.4s
+; VBITS_GE_128-NEXT: uzp1 v5.4s, v7.4s, v6.4s
+; VBITS_GE_128-NEXT: uzp1 v4.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: mov b5, v4.b[0]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[1]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldp q5, q6, [x1]
+; VBITS_GE_128-NEXT: sdiv z3.d, p1/m, z3.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[6]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[7]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #48]
+; VBITS_GE_128-NEXT: sdiv z2.d, p1/m, z2.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[4]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[5]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #32]
+; VBITS_GE_128-NEXT: sdiv z1.d, p1/m, z1.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[2]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[3]
+; VBITS_GE_128-NEXT: ushll v4.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v4.2d, v4.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; VBITS_GE_128-NEXT: sdiv z0.d, p1/m, z0.d, z6.d
+; VBITS_GE_128-NEXT: stp q1, q2, [x0, #32]
+; VBITS_GE_128-NEXT: stp q3, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: sdiv_v8i64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_256-NEXT: cmpne p2.d, p0/z, z1.d, #0
+; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: mov z3.d, p2/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: ptrue p1.s, vl4
+; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_256-NEXT: zip2 v3.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
+; VBITS_GE_256-NEXT: lsl z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: asr z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z3.d, #0
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: sdiv z0.d, p1/m, z0.d, z3.d
+; VBITS_GE_256-NEXT: zip1 v2.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z2.d, z2.s
+; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: asr z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: sdiv z1.d, p1/m, z1.d, z2.d
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: sdiv_v8i64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_512-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <8 x i64>, ptr %a
+ %op2 = load <8 x i64>, ptr %b
+ %mask = icmp ne <8 x i64> %op1, zeroinitializer
+ %res = call <8 x i64> @llvm.masked.sdiv(<8 x i64> %op1, <8 x i64> %op2, <8 x i1> %mask)
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: sdiv_v16i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <16 x i64>, ptr %a
+ %op2 = load <16 x i64>, ptr %b
+ %mask = icmp ne <16 x i64> %op1, zeroinitializer
+ %res = call <16 x i64> @llvm.masked.sdiv(<16 x i64> %op1, <16 x i64> %op2, <16 x i1> %mask)
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @sdiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: sdiv_v32i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sdiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i64>, ptr %a
+ %op2 = load <32 x i64>, ptr %b
+ %mask = icmp ne <32 x i64> %op1, zeroinitializer
+ %res = call <32 x i64> @llvm.masked.sdiv(<32 x i64> %op1, <32 x i64> %op2, <32 x i1> %mask)
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+;
+; UDIV
+;
+
+; Vector vXi8 udiv are not legal for NEON so use SVE when available.
+define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: udiv_v8i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: ushll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; VBITS_GE_128-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v8i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b
+; VBITS_GE_256-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v8i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b
+; VBITS_GE_512-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: uunpklo z1.h, z1.b
+; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: xtn v0.8b, v0.8h
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i8> %op1, zeroinitializer
+ %res = call <8 x i8> @llvm.masked.udiv(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask)
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: udiv_v16i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: ushll2 v2.8h, v0.16b, #0
+; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v3.8h, v1.16b, #0
+; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v1.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h
+; VBITS_GE_128-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v16i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: uunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: uunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16
+; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h
+; VBITS_GE_256-NEXT: uzp1 z0.b, z1.b, z1.b
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v16i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: uunpklo z1.h, z1.b
+; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <16 x i8> %op1, zeroinitializer
+ %res = call <16 x i8> @llvm.masked.udiv(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
+ ret <16 x i8> %res
+}
+
+define void @udiv_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: udiv_v32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.h, z1.b
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i8>, ptr %a
+ %op2 = load <32 x i8>, ptr %b
+ %mask = icmp ne <32 x i8> %op1, zeroinitializer
+ %res = call <32 x i8> @llvm.masked.udiv(<32 x i8> %op1, <32 x i8> %op2, <32 x i1> %mask)
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl64
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.h, z1.b
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i8>, ptr %a
+ %op2 = load <64 x i8>, ptr %b
+ %mask = icmp ne <64 x i8> %op1, zeroinitializer
+ %res = call <64 x i8> @llvm.masked.udiv(<64 x i8> %op1, <64 x i8> %op2, <64 x i1> %mask)
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v128i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: uunpklo z2.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: uunpklo z1.h, z1.b
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z3.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: st1b { z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i8>, ptr %a
+ %op2 = load <128 x i8>, ptr %b
+ %mask = icmp ne <128 x i8> %op1, zeroinitializer
+ %res = call <128 x i8> @llvm.masked.udiv(<128 x i8> %op1, <128 x i8> %op2, <128 x i1> %mask)
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v256i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: uunpklo z2.h, z0.b
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z3.h, z1.b
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: uunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: uunpklo z1.h, z1.b
+; CHECK-NEXT: uunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: udiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: uunpklo z5.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uunpklo z3.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: udiv z3.s, p1/m, z3.s, z5.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h
+; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b
+; CHECK-NEXT: ptrue p1.b, vl128
+; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b
+; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <256 x i8>, ptr %a
+ %op2 = load <256 x i8>, ptr %b
+ %mask = icmp ne <256 x i8> %op1, zeroinitializer
+ %res = call <256 x i8> @llvm.masked.udiv(<256 x i8> %op1, <256 x i8> %op2, <256 x i1> %mask)
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+; Vector vXi16 udiv are not legal for NEON so use SVE when available.
+define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
+; CHECK-LABEL: udiv_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.4h, v1.4h, v2.4h
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i16> %op1, zeroinitializer
+ %res = call <4 x i16> @llvm.masked.udiv(<4 x i16> %op1, <4 x i16> %op2, <4 x i1> %mask)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
+; VBITS_GE_128-LABEL: udiv_v8i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_128-NEXT: ushll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v8i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v8i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i16> %op1, zeroinitializer
+ %res = call <8 x i16> @llvm.masked.udiv(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask)
+ ret <8 x i16> %res
+}
+
+define void @udiv_v16i16(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: udiv_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldr q3, [x1, #16]
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: cmeq v5.8h, v1.8h, #0
+; VBITS_GE_128-NEXT: bic v3.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: ldr q4, [x1]
+; VBITS_GE_128-NEXT: bic v4.16b, v4.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v4.8h, v4.8h, v5.8h
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: ushll2 v6.4s, v4.8h, #0
+; VBITS_GE_128-NEXT: ushll v4.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: udiv z5.s, p0/m, z5.s, z6.s
+; VBITS_GE_128-NEXT: udiv z1.s, p0/m, z1.s, z4.s
+; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z2.s
+; VBITS_GE_128-NEXT: uzp1 v1.8h, v1.8h, v5.8h
+; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov z1.h, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_256-NEXT: mov z1.h, p1/m, z2.h
+; VBITS_GE_256-NEXT: uunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: uunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16
+; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ptrue p1.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h
+; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v16i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl16
+; VBITS_GE_512-NEXT: mov z2.h, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: sel z1.h, p1, z1.h, z2.h
+; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h
+; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: st1h { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i16>, ptr %a
+ %op2 = load <16 x i16>, ptr %b
+ %mask = icmp ne <16 x i16> %op1, zeroinitializer
+ %res = call <16 x i16> @llvm.masked.udiv(<16 x i16> %op1, <16 x i16> %op2, <16 x i1> %mask)
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: udiv_v32i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl32
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i16>, ptr %a
+ %op2 = load <32 x i16>, ptr %b
+ %mask = icmp ne <32 x i16> %op1, zeroinitializer
+ %res = call <32 x i16> @llvm.masked.udiv(<32 x i16> %op1, <32 x i16> %op2, <32 x i1> %mask)
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v64i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i16>, ptr %a
+ %op2 = load <64 x i16>, ptr %b
+ %mask = icmp ne <64 x i16> %op1, zeroinitializer
+ %res = call <64 x i16> @llvm.masked.udiv(<64 x i16> %op1, <64 x i16> %op2, <64 x i1> %mask)
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v128i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: mov z1.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: mov z1.h, p1/m, z2.h
+; CHECK-NEXT: uunpklo z2.s, z0.h
+; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z3.s, z1.h
+; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h
+; CHECK-NEXT: st1h { z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i16>, ptr %a
+ %op2 = load <128 x i16>, ptr %b
+ %mask = icmp ne <128 x i16> %op1, zeroinitializer
+ %res = call <128 x i16> @llvm.masked.udiv(<128 x i16> %op1, <128 x i16> %op2, <128 x i1> %mask)
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+; Vector v2i32 udiv are not legal for NEON so use SVE when available.
+define <2 x i32> @udiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: udiv_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i32> %op1, zeroinitializer
+ %res = call <2 x i32> @llvm.masked.udiv(<2 x i32> %op1, <2 x i32> %op2, <2 x i1> %mask)
+ ret <2 x i32> %res
+}
+
+; Vector v4i32 udiv are not legal for NEON so use SVE when available.
+define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: udiv_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i32> %op1, zeroinitializer
+ %res = call <4 x i32> @llvm.masked.udiv(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
+ ret <4 x i32> %res
+}
+
+define void @udiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: udiv_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <8 x i32>, ptr %a
+ %op2 = load <8 x i32>, ptr %b
+ %mask = icmp ne <8 x i32> %op1, zeroinitializer
+ %res = call <8 x i32> @llvm.masked.udiv(<8 x i32> %op1, <8 x i32> %op2, <8 x i1> %mask)
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v16i32(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: udiv_v16i32:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: cmeq v2.4s, v0.4s, #0
+; VBITS_GE_128-NEXT: cmeq v3.4s, v1.4s, #0
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: ldp q3, q4, [x0, #32]
+; VBITS_GE_128-NEXT: cmeq v5.4s, v4.4s, #0
+; VBITS_GE_128-NEXT: cmeq v6.4s, v3.4s, #0
+; VBITS_GE_128-NEXT: mvn v2.16b, v2.16b
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: zip1 v7.8b, v2.8b, v0.8b
+; VBITS_GE_128-NEXT: zip2 v2.8b, v2.8b, v0.8b
+; VBITS_GE_128-NEXT: xtn v5.8b, v5.8h
+; VBITS_GE_128-NEXT: ushll v6.4s, v7.4h, #0
+; VBITS_GE_128-NEXT: ldp q7, q16, [x1]
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v2.4s, v2.4s, #31
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z6.s, #0
+; VBITS_GE_128-NEXT: zip2 v6.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: zip1 v5.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: ushll v6.4s, v6.4h, #0
+; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: udiv z1.s, p1/m, z1.s, z7.s
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v5.4s, v5.4s, #31
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z6.s, #0
+; VBITS_GE_128-NEXT: ldr q6, [x1, #48]
+; VBITS_GE_128-NEXT: udiv z4.s, p1/m, z4.s, z6.s
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z5.s, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #32]
+; VBITS_GE_128-NEXT: udiv z3.s, p1/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: cmpne p1.s, p0/z, z2.s, #0
+; VBITS_GE_128-NEXT: udiv z0.s, p1/m, z0.s, z16.s
+; VBITS_GE_128-NEXT: stp q3, q4, [x0, #32]
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v16i32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_256-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; VBITS_GE_256-NEXT: udiv z1.s, p1/m, z1.s, z2.s
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v16i32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_512-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i32>, ptr %a
+ %op2 = load <16 x i32>, ptr %b
+ %mask = icmp ne <16 x i32> %op1, zeroinitializer
+ %res = call <16 x i32> @llvm.masked.udiv(<16 x i32> %op1, <16 x i32> %op2, <16 x i1> %mask)
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: udiv_v32i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i32>, ptr %a
+ %op2 = load <32 x i32>, ptr %b
+ %mask = icmp ne <32 x i32> %op1, zeroinitializer
+ %res = call <32 x i32> @llvm.masked.udiv(<32 x i32> %op1, <32 x i32> %op2, <32 x i1> %mask)
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v64i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i32>, ptr %a
+ %op2 = load <64 x i32>, ptr %b
+ %mask = icmp ne <64 x i32> %op1, zeroinitializer
+ %res = call <64 x i32> @llvm.masked.udiv(<64 x i32> %op1, <64 x i32> %op2, <64 x i1> %mask)
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+; TODO: Vector i64 udiv are not legal for NEON but the operation is scalarised
+; during type legalisation, so we don't get chance to use SVE.
+define <1 x i64> @udiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: udiv_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: fmov x9, d1
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csinc x9, x9, xzr, ne
+; CHECK-NEXT: udiv x8, x8, x9
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+ %mask = icmp ne <1 x i64> %op1, zeroinitializer
+ %res = call <1 x i64> @llvm.masked.udiv(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask)
+ ret <1 x i64> %res
+}
+
+; Vector i64 udiv are not legal for NEON so use SVE when available.
+define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: udiv_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: udiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i64> %op1, zeroinitializer
+ %res = call <2 x i64> @llvm.masked.udiv(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
+ ret <2 x i64> %res
+}
+
+define void @udiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: udiv_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: udiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <4 x i64>, ptr %a
+ %op2 = load <4 x i64>, ptr %b
+ %mask = icmp ne <4 x i64> %op1, zeroinitializer
+ %res = call <4 x i64> @llvm.masked.udiv(<4 x i64> %op1, <4 x i64> %op2, <4 x i1> %mask)
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v8i64(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: udiv_v8i64:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q2, [x0, #32]
+; VBITS_GE_128-NEXT: ptrue p0.d, vl2
+; VBITS_GE_128-NEXT: ldp q3, q0, [x0]
+; VBITS_GE_128-NEXT: cmeq v4.2d, v2.2d, #0
+; VBITS_GE_128-NEXT: cmeq v5.2d, v1.2d, #0
+; VBITS_GE_128-NEXT: cmeq v6.2d, v0.2d, #0
+; VBITS_GE_128-NEXT: cmeq v7.2d, v3.2d, #0
+; VBITS_GE_128-NEXT: uzp1 v4.4s, v5.4s, v4.4s
+; VBITS_GE_128-NEXT: uzp1 v5.4s, v7.4s, v6.4s
+; VBITS_GE_128-NEXT: uzp1 v4.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: mov b5, v4.b[0]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[1]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldp q5, q6, [x1]
+; VBITS_GE_128-NEXT: udiv z3.d, p1/m, z3.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[6]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[7]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #48]
+; VBITS_GE_128-NEXT: udiv z2.d, p1/m, z2.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[4]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[5]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z5.d, #0
+; VBITS_GE_128-NEXT: ldr q5, [x1, #32]
+; VBITS_GE_128-NEXT: udiv z1.d, p1/m, z1.d, z5.d
+; VBITS_GE_128-NEXT: mov b5, v4.b[2]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[3]
+; VBITS_GE_128-NEXT: ushll v4.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v4.2d, v4.2d, #63
+; VBITS_GE_128-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; VBITS_GE_128-NEXT: udiv z0.d, p1/m, z0.d, z6.d
+; VBITS_GE_128-NEXT: stp q1, q2, [x0, #32]
+; VBITS_GE_128-NEXT: stp q3, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: udiv_v8i64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_256-NEXT: cmpne p2.d, p0/z, z1.d, #0
+; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: mov z3.d, p2/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: ptrue p1.s, vl4
+; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_256-NEXT: zip2 v3.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
+; VBITS_GE_256-NEXT: lsl z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: asr z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z3.d, #0
+; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: udiv z0.d, p1/m, z0.d, z3.d
+; VBITS_GE_256-NEXT: zip1 v2.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z2.d, z2.s
+; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: asr z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: udiv z1.d, p1/m, z1.d, z2.d
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: udiv_v8i64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_512-NEXT: udiv z0.d, p1/m, z0.d, z1.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <8 x i64>, ptr %a
+ %op2 = load <8 x i64>, ptr %b
+ %mask = icmp ne <8 x i64> %op1, zeroinitializer
+ %res = call <8 x i64> @llvm.masked.udiv(<8 x i64> %op1, <8 x i64> %op2, <8 x i1> %mask)
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: udiv_v16i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: udiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <16 x i64>, ptr %a
+ %op2 = load <16 x i64>, ptr %b
+ %mask = icmp ne <16 x i64> %op1, zeroinitializer
+ %res = call <16 x i64> @llvm.masked.udiv(<16 x i64> %op1, <16 x i64> %op2, <16 x i1> %mask)
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @udiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: udiv_v32i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: udiv z0.d, p1/m, z0.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i64>, ptr %a
+ %op2 = load <32 x i64>, ptr %b
+ %mask = icmp ne <32 x i64> %op1, zeroinitializer
+ %res = call <32 x i64> @llvm.masked.udiv(<32 x i64> %op1, <32 x i64> %op2, <32 x i1> %mask)
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-rem.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-rem.ll
new file mode 100644
index 0000000000000..f19e95b6df627
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-rem.ll
@@ -0,0 +1,1957 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_128
+; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; SREM
+;
+
+; Vector vXi8 srem are not legal for NEON so use SVE when available.
+define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: srem_v8i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sshll v2.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: sshll v3.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v8i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: sunpklo z2.h, z0.b
+; VBITS_GE_256-NEXT: sunpklo z3.h, z1.b
+; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_256-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v8i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: sunpklo z2.h, z0.b
+; VBITS_GE_512-NEXT: sunpklo z3.h, z1.b
+; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
+; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h
+; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_512-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i8> %op1, zeroinitializer
+ %res = call <8 x i8> @llvm.masked.srem(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask)
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: srem_v16i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sshll2 v2.8h, v0.16b, #0
+; VBITS_GE_128-NEXT: sshll2 v3.8h, v1.16b, #0
+; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: sshll v5.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v7.4s, v5.8h, #0
+; VBITS_GE_128-NEXT: sshll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: sshll v3.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: sdiv z6.s, p0/m, z6.s, z7.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h
+; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v16i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sunpklo z2.h, z0.b
+; VBITS_GE_256-NEXT: sunpklo z3.h, z1.b
+; VBITS_GE_256-NEXT: sunpklo z4.s, z2.h
+; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16
+; VBITS_GE_256-NEXT: sunpklo z5.s, z3.h
+; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16
+; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: sdiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_256-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b
+; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v16i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sunpklo z2.h, z0.b
+; VBITS_GE_512-NEXT: sunpklo z3.h, z1.b
+; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h
+; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h
+; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <16 x i8> %op1, zeroinitializer
+ %res = call <16 x i8> @llvm.masked.srem(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
+ ret <16 x i8> %res
+}
+
+define void @srem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: srem_v32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: sunpklo z3.h, z0.b
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: ptrue p1.s, vl32
+; CHECK-NEXT: sunpklo z2.h, z1.b
+; CHECK-NEXT: sunpklo z2.s, z2.h
+; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i8>, ptr %a
+ %op2 = load <32 x i8>, ptr %b
+ %mask = icmp ne <32 x i8> %op1, zeroinitializer
+ %res = call <32 x i8> @llvm.masked.srem(<32 x i8> %op1, <32 x i8> %op2, <32 x i1> %mask)
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @srem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl64
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: sunpklo z3.h, z0.b
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z2.h, z1.b
+; CHECK-NEXT: sunpklo z2.s, z2.h
+; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i8>, ptr %a
+ %op2 = load <64 x i8>, ptr %b
+ %mask = icmp ne <64 x i8> %op1, zeroinitializer
+ %res = call <64 x i8> @llvm.masked.srem(<64 x i8> %op1, <64 x i8> %op2, <64 x i1> %mask)
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @srem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v128i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: sunpklo z2.h, z0.b
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z3.h, z1.b
+; CHECK-NEXT: sunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: sunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: sunpklo z2.s, z2.h
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: sdiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i8>, ptr %a
+ %op2 = load <128 x i8>, ptr %b
+ %mask = icmp ne <128 x i8> %op1, zeroinitializer
+ %res = call <128 x i8> @llvm.masked.srem(<128 x i8> %op1, <128 x i8> %op2, <128 x i1> %mask)
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v256i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: sunpklo z2.h, z0.b
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z3.h, z1.b
+; CHECK-NEXT: sunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: sunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: sunpklo z2.s, z2.h
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: sdiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: movprfx z5, z1
+; CHECK-NEXT: ext z5.b, z5.b, z1.b, #128
+; CHECK-NEXT: sunpklo z5.h, z5.b
+; CHECK-NEXT: sunpklo z7.s, z5.h
+; CHECK-NEXT: ext z5.b, z5.b, z5.b, #128
+; CHECK-NEXT: sunpklo z5.s, z5.h
+; CHECK-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: ext z3.b, z3.b, z0.b, #128
+; CHECK-NEXT: sunpklo z3.h, z3.b
+; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h
+; CHECK-NEXT: sunpklo z6.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: sdiv z6.s, p1/m, z6.s, z7.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: sdiv z3.s, p1/m, z3.s, z5.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h
+; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h
+; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h
+; CHECK-NEXT: ptrue p1.b, vl128
+; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b
+; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <256 x i8>, ptr %a
+ %op2 = load <256 x i8>, ptr %b
+ %mask = icmp ne <256 x i8> %op1, zeroinitializer
+ %res = call <256 x i8> @llvm.masked.srem(<256 x i8> %op1, <256 x i8> %op2, <256 x i1> %mask)
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+; Vector vXi16 srem are not legal for NEON so use SVE when available.
+define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
+; CHECK-LABEL: srem_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.4h, v1.4h, v2.4h
+; CHECK-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT: xtn v2.4h, v2.4s
+; CHECK-NEXT: mls v0.4h, v2.4h, v1.4h
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i16> %op1, zeroinitializer
+ %res = call <4 x i16> @llvm.masked.srem(<4 x i16> %op1, <4 x i16> %op2, <4 x i1> %mask)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
+; VBITS_GE_128-LABEL: srem_v8i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_128-NEXT: sshll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll v4.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: sshll v3.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v8i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: sunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v8i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_512-NEXT: sunpklo z2.s, z0.h
+; VBITS_GE_512-NEXT: sunpklo z3.s, z1.h
+; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i16> %op1, zeroinitializer
+ %res = call <8 x i16> @llvm.masked.srem(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask)
+ ret <8 x i16> %res
+}
+
+define void @srem_v16i16(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: srem_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldr q3, [x1, #16]
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: cmeq v5.8h, v1.8h, #0
+; VBITS_GE_128-NEXT: bic v3.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: sshll v16.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: ldr q4, [x1]
+; VBITS_GE_128-NEXT: bic v4.16b, v4.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v4.8h, v4.8h, v5.8h
+; VBITS_GE_128-NEXT: sshll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: sshll2 v6.4s, v4.8h, #0
+; VBITS_GE_128-NEXT: sshll v7.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: sdiv z5.s, p0/m, z5.s, z6.s
+; VBITS_GE_128-NEXT: sshll v6.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: sdiv z6.s, p0/m, z6.s, z7.s
+; VBITS_GE_128-NEXT: sshll v7.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: sdiv z7.s, p0/m, z7.s, z16.s
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: mls v1.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v3.8h, v7.8h, v3.8h
+; VBITS_GE_128-NEXT: mls v0.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov z1.h, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_256-NEXT: mov z1.h, p1/m, z2.h
+; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: sunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: movprfx z4, z1
+; VBITS_GE_256-NEXT: ext z4.b, z4.b, z1.b, #16
+; VBITS_GE_256-NEXT: sunpklo z4.s, z4.h
+; VBITS_GE_256-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: movprfx z3, z0
+; VBITS_GE_256-NEXT: ext z3.b, z3.b, z0.b, #16
+; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: sdiv z3.s, p1/m, z3.s, z4.s
+; VBITS_GE_256-NEXT: ptrue p1.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h
+; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v16i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl16
+; VBITS_GE_512-NEXT: mov z2.h, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h
+; VBITS_GE_512-NEXT: sel z1.h, p1, z1.h, z2.h
+; VBITS_GE_512-NEXT: ptrue p1.s, vl16
+; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h
+; VBITS_GE_512-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i16>, ptr %a
+ %op2 = load <16 x i16>, ptr %b
+ %mask = icmp ne <16 x i16> %op1, zeroinitializer
+ %res = call <16 x i16> @llvm.masked.srem(<16 x i16> %op1, <16 x i16> %op2, <16 x i1> %mask)
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @srem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: srem_v32i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl32
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: sunpklo z3.s, z0.h
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: ptrue p1.s, vl32
+; CHECK-NEXT: sunpklo z2.s, z1.h
+; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i16>, ptr %a
+ %op2 = load <32 x i16>, ptr %b
+ %mask = icmp ne <32 x i16> %op1, zeroinitializer
+ %res = call <32 x i16> @llvm.masked.srem(<32 x i16> %op1, <32 x i16> %op2, <32 x i1> %mask)
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @srem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v64i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: sunpklo z3.s, z0.h
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z2.s, z1.h
+; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i16>, ptr %a
+ %op2 = load <64 x i16>, ptr %b
+ %mask = icmp ne <64 x i16> %op1, zeroinitializer
+ %res = call <64 x i16> @llvm.masked.srem(<64 x i16> %op1, <64 x i16> %op2, <64 x i1> %mask)
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @srem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v128i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: mov z1.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: mov z1.h, p1/m, z2.h
+; CHECK-NEXT: sunpklo z2.s, z0.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: sunpklo z3.s, z1.h
+; CHECK-NEXT: movprfx z4, z1
+; CHECK-NEXT: ext z4.b, z4.b, z1.b, #128
+; CHECK-NEXT: sunpklo z4.s, z4.h
+; CHECK-NEXT: sdiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: ext z3.b, z3.b, z0.b, #128
+; CHECK-NEXT: sunpklo z3.s, z3.h
+; CHECK-NEXT: sdiv z3.s, p1/m, z3.s, z4.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i16>, ptr %a
+ %op2 = load <128 x i16>, ptr %b
+ %mask = icmp ne <128 x i16> %op1, zeroinitializer
+ %res = call <128 x i16> @llvm.masked.srem(<128 x i16> %op1, <128 x i16> %op2, <128 x i1> %mask)
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+; Vector v2i32 srem are not legal for NEON so use SVE when available.
+define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: srem_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: cmeq v2.2s, v0.2s, #0
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.2s, v1.2s, v2.2s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i32> %op1, zeroinitializer
+ %res = call <2 x i32> @llvm.masked.srem(<2 x i32> %op1, <2 x i32> %op2, <2 x i1> %mask)
+ ret <2 x i32> %res
+}
+
+; Vector v4i32 srem are not legal for NEON so use SVE when available.
+define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: srem_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmeq v2.4s, v0.4s, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: sub v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i32> %op1, zeroinitializer
+ %res = call <4 x i32> @llvm.masked.srem(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
+ ret <4 x i32> %res
+}
+
+define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: srem_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <8 x i32>, ptr %a
+ %op2 = load <8 x i32>, ptr %b
+ %mask = icmp ne <8 x i32> %op1, zeroinitializer
+ %res = call <8 x i32> @llvm.masked.srem(<8 x i32> %op1, <8 x i32> %op2, <8 x i1> %mask)
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @srem_v16i32(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: srem_v16i32:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldp q16, q17, [x1]
+; VBITS_GE_128-NEXT: ldp q19, q18, [x1, #32]
+; VBITS_GE_128-NEXT: cmeq v2.4s, v1.4s, #0
+; VBITS_GE_128-NEXT: cmeq v3.4s, v0.4s, #0
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v2.16b
+; VBITS_GE_128-NEXT: ldp q2, q3, [x0, #32]
+; VBITS_GE_128-NEXT: cmeq v5.4s, v3.4s, #0
+; VBITS_GE_128-NEXT: cmeq v6.4s, v2.4s, #0
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: zip1 v6.8b, v4.8b, v0.8b
+; VBITS_GE_128-NEXT: zip2 v4.8b, v4.8b, v0.8b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: ushll v6.4s, v6.4h, #0
+; VBITS_GE_128-NEXT: ushll v4.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: xtn v5.8b, v5.8h
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v4.4s, v4.4s, #31
+; VBITS_GE_128-NEXT: zip2 v7.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: zip1 v5.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: cmlt v6.4s, v6.4s, #0
+; VBITS_GE_128-NEXT: cmlt v4.4s, v4.4s, #0
+; VBITS_GE_128-NEXT: and v16.16b, v16.16b, v6.16b
+; VBITS_GE_128-NEXT: mvn v6.16b, v6.16b
+; VBITS_GE_128-NEXT: ushll v7.4s, v7.4h, #0
+; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: and v17.16b, v17.16b, v4.16b
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: sub v6.4s, v16.4s, v6.4s
+; VBITS_GE_128-NEXT: shl v7.4s, v7.4s, #31
+; VBITS_GE_128-NEXT: shl v5.4s, v5.4s, #31
+; VBITS_GE_128-NEXT: sub v4.4s, v17.4s, v4.4s
+; VBITS_GE_128-NEXT: movprfx z16, z0
+; VBITS_GE_128-NEXT: sdiv z16.s, p0/m, z16.s, z6.s
+; VBITS_GE_128-NEXT: cmlt v7.4s, v7.4s, #0
+; VBITS_GE_128-NEXT: cmlt v5.4s, v5.4s, #0
+; VBITS_GE_128-NEXT: movprfx z17, z1
+; VBITS_GE_128-NEXT: sdiv z17.s, p0/m, z17.s, z4.s
+; VBITS_GE_128-NEXT: and v18.16b, v18.16b, v7.16b
+; VBITS_GE_128-NEXT: mvn v7.16b, v7.16b
+; VBITS_GE_128-NEXT: and v19.16b, v19.16b, v5.16b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v7.4s, v18.4s, v7.4s
+; VBITS_GE_128-NEXT: sub v5.4s, v19.4s, v5.4s
+; VBITS_GE_128-NEXT: movprfx z18, z3
+; VBITS_GE_128-NEXT: sdiv z18.s, p0/m, z18.s, z7.s
+; VBITS_GE_128-NEXT: movprfx z19, z2
+; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z5.s
+; VBITS_GE_128-NEXT: mls v0.4s, v16.4s, v6.4s
+; VBITS_GE_128-NEXT: mls v1.4s, v17.4s, v4.4s
+; VBITS_GE_128-NEXT: stp q0, q1, [x0]
+; VBITS_GE_128-NEXT: mls v3.4s, v18.4s, v7.4s
+; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v5.4s
+; VBITS_GE_128-NEXT: stp q2, q3, [x0, #32]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v16i32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: mov z1.s, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z5.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_256-NEXT: sel z2.s, p1, z2.s, z1.s
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z4.s, #0
+; VBITS_GE_256-NEXT: movprfx z3, z0
+; VBITS_GE_256-NEXT: sdiv z3.s, p0/m, z3.s, z2.s
+; VBITS_GE_256-NEXT: mov z1.s, p1/m, z5.s
+; VBITS_GE_256-NEXT: movprfx z5, z4
+; VBITS_GE_256-NEXT: sdiv z5.s, p0/m, z5.s, z1.s
+; VBITS_GE_256-NEXT: mls z0.s, p0/m, z3.s, z2.s
+; VBITS_GE_256-NEXT: msb z1.s, p0/m, z5.s, z4.s
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v16i32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: mov z2.s, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_512-NEXT: sel z1.s, p1, z1.s, z2.s
+; VBITS_GE_512-NEXT: movprfx z2, z0
+; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i32>, ptr %a
+ %op2 = load <16 x i32>, ptr %b
+ %mask = icmp ne <16 x i32> %op1, zeroinitializer
+ %res = call <16 x i32> @llvm.masked.srem(<16 x i32> %op1, <16 x i32> %op2, <16 x i1> %mask)
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @srem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: srem_v32i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i32>, ptr %a
+ %op2 = load <32 x i32>, ptr %b
+ %mask = icmp ne <32 x i32> %op1, zeroinitializer
+ %res = call <32 x i32> @llvm.masked.srem(<32 x i32> %op1, <32 x i32> %op2, <32 x i1> %mask)
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @srem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v64i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i32>, ptr %a
+ %op2 = load <64 x i32>, ptr %b
+ %mask = icmp ne <64 x i32> %op1, zeroinitializer
+ %res = call <64 x i32> @llvm.masked.srem(<64 x i32> %op1, <64 x i32> %op2, <64 x i1> %mask)
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+; Vector i64 srem are not legal for NEON so use SVE when available.
+define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: srem_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: fmov x9, d1
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csinc x9, x9, xzr, ne
+; CHECK-NEXT: sdiv x10, x8, x9
+; CHECK-NEXT: msub x8, x10, x9, x8
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+ %mask = icmp ne <1 x i64> %op1, zeroinitializer
+ %res = call <1 x i64> @llvm.masked.srem(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask)
+ ret <1 x i64> %res
+}
+
+; Vector i64 srem are not legal for NEON so use SVE when available.
+define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: srem_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmeq v2.2d, v0.2d, #0
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: bic v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: sub v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i64> %op1, zeroinitializer
+ %res = call <2 x i64> @llvm.masked.srem(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
+ ret <2 x i64> %res
+}
+
+define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: srem_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <4 x i64>, ptr %a
+ %op2 = load <4 x i64>, ptr %b
+ %mask = icmp ne <4 x i64> %op1, zeroinitializer
+ %res = call <4 x i64> @llvm.masked.srem(<4 x i64> %op1, <4 x i64> %op2, <4 x i1> %mask)
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @srem_v8i64(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: srem_v8i64:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0, #32]
+; VBITS_GE_128-NEXT: ptrue p0.d, vl2
+; VBITS_GE_128-NEXT: ldp q2, q3, [x0]
+; VBITS_GE_128-NEXT: ldp q19, q17, [x1, #32]
+; VBITS_GE_128-NEXT: cmeq v4.2d, v0.2d, #0
+; VBITS_GE_128-NEXT: cmeq v5.2d, v1.2d, #0
+; VBITS_GE_128-NEXT: cmeq v6.2d, v3.2d, #0
+; VBITS_GE_128-NEXT: cmeq v7.2d, v2.2d, #0
+; VBITS_GE_128-NEXT: uzp1 v4.4s, v5.4s, v4.4s
+; VBITS_GE_128-NEXT: uzp1 v5.4s, v7.4s, v6.4s
+; VBITS_GE_128-NEXT: ldp q6, q7, [x1]
+; VBITS_GE_128-NEXT: uzp1 v4.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: mov b5, v4.b[0]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[1]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmlt v5.2d, v5.2d, #0
+; VBITS_GE_128-NEXT: and v6.16b, v6.16b, v5.16b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v5.2d, v6.2d, v5.2d
+; VBITS_GE_128-NEXT: mov b6, v4.b[6]
+; VBITS_GE_128-NEXT: movprfx z16, z2
+; VBITS_GE_128-NEXT: sdiv z16.d, p0/m, z16.d, z5.d
+; VBITS_GE_128-NEXT: mov v6.b[4], v4.b[7]
+; VBITS_GE_128-NEXT: ushll v6.2d, v6.2s, #0
+; VBITS_GE_128-NEXT: shl v6.2d, v6.2d, #63
+; VBITS_GE_128-NEXT: cmlt v6.2d, v6.2d, #0
+; VBITS_GE_128-NEXT: and v17.16b, v17.16b, v6.16b
+; VBITS_GE_128-NEXT: mvn v6.16b, v6.16b
+; VBITS_GE_128-NEXT: sub v6.2d, v17.2d, v6.2d
+; VBITS_GE_128-NEXT: mov b17, v4.b[4]
+; VBITS_GE_128-NEXT: movprfx z18, z0
+; VBITS_GE_128-NEXT: sdiv z18.d, p0/m, z18.d, z6.d
+; VBITS_GE_128-NEXT: mov v17.b[4], v4.b[5]
+; VBITS_GE_128-NEXT: ushll v17.2d, v17.2s, #0
+; VBITS_GE_128-NEXT: mls z2.d, p0/m, z16.d, z5.d
+; VBITS_GE_128-NEXT: shl v17.2d, v17.2d, #63
+; VBITS_GE_128-NEXT: cmlt v17.2d, v17.2d, #0
+; VBITS_GE_128-NEXT: and v19.16b, v19.16b, v17.16b
+; VBITS_GE_128-NEXT: mvn v17.16b, v17.16b
+; VBITS_GE_128-NEXT: sub v17.2d, v19.2d, v17.2d
+; VBITS_GE_128-NEXT: mov b19, v4.b[2]
+; VBITS_GE_128-NEXT: movprfx z20, z1
+; VBITS_GE_128-NEXT: sdiv z20.d, p0/m, z20.d, z17.d
+; VBITS_GE_128-NEXT: mov v19.b[4], v4.b[3]
+; VBITS_GE_128-NEXT: ushll v4.2d, v19.2s, #0
+; VBITS_GE_128-NEXT: mls z0.d, p0/m, z18.d, z6.d
+; VBITS_GE_128-NEXT: shl v4.2d, v4.2d, #63
+; VBITS_GE_128-NEXT: cmlt v4.2d, v4.2d, #0
+; VBITS_GE_128-NEXT: and v7.16b, v7.16b, v4.16b
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: sub v4.2d, v7.2d, v4.2d
+; VBITS_GE_128-NEXT: movprfx z7, z3
+; VBITS_GE_128-NEXT: sdiv z7.d, p0/m, z7.d, z4.d
+; VBITS_GE_128-NEXT: mls z1.d, p0/m, z20.d, z17.d
+; VBITS_GE_128-NEXT: stp q1, q0, [x0, #32]
+; VBITS_GE_128-NEXT: mls z3.d, p0/m, z7.d, z4.d
+; VBITS_GE_128-NEXT: stp q2, q3, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: srem_v8i64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_256-NEXT: cmpne p2.d, p0/z, z1.d, #0
+; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: mov z3.d, p2/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: ptrue p1.s, vl4
+; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
+; VBITS_GE_256-NEXT: ptrue p1.d
+; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_256-NEXT: zip2 v3.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: zip1 v2.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
+; VBITS_GE_256-NEXT: uunpklo z2.d, z2.s
+; VBITS_GE_256-NEXT: lsl z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: asr z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: asr z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: and z3.d, z3.d, #0x1
+; VBITS_GE_256-NEXT: and z2.d, z2.d, #0x1
+; VBITS_GE_256-NEXT: cmpne p2.d, p1/z, z3.d, #0
+; VBITS_GE_256-NEXT: mov z3.d, #1 // =0x1
+; VBITS_GE_256-NEXT: sel z4.d, p2, z4.d, z3.d
+; VBITS_GE_256-NEXT: cmpne p2.d, p1/z, z2.d, #0
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: movprfx z5, z0
+; VBITS_GE_256-NEXT: sdiv z5.d, p0/m, z5.d, z4.d
+; VBITS_GE_256-NEXT: sel z2.d, p2, z2.d, z3.d
+; VBITS_GE_256-NEXT: movprfx z3, z1
+; VBITS_GE_256-NEXT: sdiv z3.d, p0/m, z3.d, z2.d
+; VBITS_GE_256-NEXT: mls z0.d, p0/m, z5.d, z4.d
+; VBITS_GE_256-NEXT: mls z1.d, p0/m, z3.d, z2.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: srem_v8i64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: mov z2.d, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_512-NEXT: sel z1.d, p1, z1.d, z2.d
+; VBITS_GE_512-NEXT: movprfx z2, z0
+; VBITS_GE_512-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
+; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <8 x i64>, ptr %a
+ %op2 = load <8 x i64>, ptr %b
+ %mask = icmp ne <8 x i64> %op1, zeroinitializer
+ %res = call <8 x i64> @llvm.masked.srem(<8 x i64> %op1, <8 x i64> %op2, <8 x i1> %mask)
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @srem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: srem_v16i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <16 x i64>, ptr %a
+ %op2 = load <16 x i64>, ptr %b
+ %mask = icmp ne <16 x i64> %op1, zeroinitializer
+ %res = call <16 x i64> @llvm.masked.srem(<16 x i64> %op1, <16 x i64> %op2, <16 x i1> %mask)
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @srem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: srem_v32i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i64>, ptr %a
+ %op2 = load <32 x i64>, ptr %b
+ %mask = icmp ne <32 x i64> %op1, zeroinitializer
+ %res = call <32 x i64> @llvm.masked.srem(<32 x i64> %op1, <32 x i64> %op2, <32 x i1> %mask)
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+;
+; UREM
+;
+
+; Vector vXi8 urem are not legal for NEON so use SVE when available.
+define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: urem_v8i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_128-NEXT: ushll v2.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: ushll v3.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v8i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_256-NEXT: uunpklo z2.h, z0.b
+; VBITS_GE_256-NEXT: uunpklo z3.h, z1.b
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_256-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v8i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8b, v0.8b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: bic v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: sub v1.8b, v1.8b, v2.8b
+; VBITS_GE_512-NEXT: uunpklo z2.h, z0.b
+; VBITS_GE_512-NEXT: uunpklo z3.h, z1.b
+; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: xtn v2.8b, v2.8h
+; VBITS_GE_512-NEXT: mls v0.8b, v2.8b, v1.8b
+; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i8> %op1, zeroinitializer
+ %res = call <8 x i8> @llvm.masked.urem(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask)
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
+; VBITS_GE_128-LABEL: urem_v16i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: ushll2 v2.8h, v0.16b, #0
+; VBITS_GE_128-NEXT: ushll2 v3.8h, v1.16b, #0
+; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_128-NEXT: ushll v5.8h, v1.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v7.4s, v5.8h, #0
+; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: ushll v3.8h, v0.8b, #0
+; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #0
+; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0
+; VBITS_GE_128-NEXT: udiv z6.s, p0/m, z6.s, z7.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h
+; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z5.s
+; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h
+; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v16i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: uunpklo z2.h, z0.b
+; VBITS_GE_256-NEXT: uunpklo z3.h, z1.b
+; VBITS_GE_256-NEXT: uunpklo z4.s, z2.h
+; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16
+; VBITS_GE_256-NEXT: uunpklo z5.s, z3.h
+; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: udiv z4.s, p0/m, z4.s, z5.s
+; VBITS_GE_256-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b
+; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v16i8:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.16b, v0.16b, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: uunpklo z2.h, z0.b
+; VBITS_GE_512-NEXT: uunpklo z3.h, z1.b
+; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <16 x i8> %op1, zeroinitializer
+ %res = call <16 x i8> @llvm.masked.urem(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
+ ret <16 x i8> %res
+}
+
+define void @urem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: urem_v32i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: uunpklo z3.h, z0.b
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: ptrue p1.s, vl32
+; CHECK-NEXT: uunpklo z2.h, z1.b
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i8>, ptr %a
+ %op2 = load <32 x i8>, ptr %b
+ %mask = icmp ne <32 x i8> %op1, zeroinitializer
+ %res = call <32 x i8> @llvm.masked.urem(<32 x i8> %op1, <32 x i8> %op2, <32 x i1> %mask)
+ store <32 x i8> %res, ptr %a
+ ret void
+}
+
+define void @urem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v64i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl64
+; CHECK-NEXT: mov z2.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: uunpklo z3.h, z0.b
+; CHECK-NEXT: sel z1.b, p1, z1.b, z2.b
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z2.h, z1.b
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i8>, ptr %a
+ %op2 = load <64 x i8>, ptr %b
+ %mask = icmp ne <64 x i8> %op1, zeroinitializer
+ %res = call <64 x i8> @llvm.masked.urem(<64 x i8> %op1, <64 x i8> %op2, <64 x i1> %mask)
+ store <64 x i8> %res, ptr %a
+ ret void
+}
+
+define void @urem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v128i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl128
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: uunpklo z2.h, z0.b
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z3.h, z1.b
+; CHECK-NEXT: uunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: uunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: udiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h
+; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i8>, ptr %a
+ %op2 = load <128 x i8>, ptr %b
+ %mask = icmp ne <128 x i8> %op1, zeroinitializer
+ %res = call <128 x i8> @llvm.masked.urem(<128 x i8> %op1, <128 x i8> %op2, <128 x i1> %mask)
+ store <128 x i8> %res, ptr %a
+ ret void
+}
+
+define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v256i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl256
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: mov z1.b, p1/m, z2.b
+; CHECK-NEXT: uunpklo z2.h, z0.b
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z3.h, z1.b
+; CHECK-NEXT: uunpklo z4.s, z2.h
+; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128
+; CHECK-NEXT: uunpklo z5.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: udiv z4.s, p1/m, z4.s, z5.s
+; CHECK-NEXT: movprfx z5, z1
+; CHECK-NEXT: ext z5.b, z5.b, z1.b, #128
+; CHECK-NEXT: uunpklo z5.h, z5.b
+; CHECK-NEXT: uunpklo z7.s, z5.h
+; CHECK-NEXT: ext z5.b, z5.b, z5.b, #128
+; CHECK-NEXT: uunpklo z5.s, z5.h
+; CHECK-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: ext z3.b, z3.b, z0.b, #128
+; CHECK-NEXT: uunpklo z3.h, z3.b
+; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h
+; CHECK-NEXT: uunpklo z6.s, z3.h
+; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: udiv z6.s, p1/m, z6.s, z7.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: udiv z3.s, p1/m, z3.s, z5.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h
+; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h
+; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h
+; CHECK-NEXT: ptrue p1.b, vl128
+; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b
+; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b
+; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <256 x i8>, ptr %a
+ %op2 = load <256 x i8>, ptr %b
+ %mask = icmp ne <256 x i8> %op1, zeroinitializer
+ %res = call <256 x i8> @llvm.masked.urem(<256 x i8> %op1, <256 x i8> %op2, <256 x i1> %mask)
+ store <256 x i8> %res, ptr %a
+ ret void
+}
+
+; Vector vXi16 urem are not legal for NEON so use SVE when available.
+define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
+; CHECK-LABEL: urem_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v2.4h, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.4h, v1.4h, v2.4h
+; CHECK-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT: xtn v2.4h, v2.4s
+; CHECK-NEXT: mls v0.4h, v2.4h, v1.4h
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i16> %op1, zeroinitializer
+ %res = call <4 x i16> @llvm.masked.urem(<4 x i16> %op1, <4 x i16> %op2, <4 x i1> %mask)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
+; VBITS_GE_128-LABEL: urem_v8i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_128-NEXT: ushll2 v2.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll v4.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_128-NEXT: ushll v3.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v8i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_256-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_256-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_256-NEXT: uunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: uunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v8i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0
+; VBITS_GE_512-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_512-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512-NEXT: bic v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: sub v1.8h, v1.8h, v2.8h
+; VBITS_GE_512-NEXT: uunpklo z2.s, z0.h
+; VBITS_GE_512-NEXT: uunpklo z3.s, z1.h
+; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h
+; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0
+; VBITS_GE_512-NEXT: ret
+ %mask = icmp ne <8 x i16> %op1, zeroinitializer
+ %res = call <8 x i16> @llvm.masked.urem(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask)
+ ret <8 x i16> %res
+}
+
+define void @urem_v16i16(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: urem_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldr q3, [x1, #16]
+; VBITS_GE_128-NEXT: cmeq v2.8h, v0.8h, #0
+; VBITS_GE_128-NEXT: cmeq v5.8h, v1.8h, #0
+; VBITS_GE_128-NEXT: bic v3.16b, v3.16b, v2.16b
+; VBITS_GE_128-NEXT: sub v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0
+; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0
+; VBITS_GE_128-NEXT: ushll v16.4s, v2.4h, #0
+; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z4.s
+; VBITS_GE_128-NEXT: ldr q4, [x1]
+; VBITS_GE_128-NEXT: bic v4.16b, v4.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v4.8h, v4.8h, v5.8h
+; VBITS_GE_128-NEXT: ushll2 v5.4s, v1.8h, #0
+; VBITS_GE_128-NEXT: ushll2 v6.4s, v4.8h, #0
+; VBITS_GE_128-NEXT: ushll v7.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: udiv z5.s, p0/m, z5.s, z6.s
+; VBITS_GE_128-NEXT: ushll v6.4s, v1.4h, #0
+; VBITS_GE_128-NEXT: udiv z6.s, p0/m, z6.s, z7.s
+; VBITS_GE_128-NEXT: ushll v7.4s, v0.4h, #0
+; VBITS_GE_128-NEXT: udiv z7.s, p0/m, z7.s, z16.s
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: mls v1.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v3.8h, v7.8h, v3.8h
+; VBITS_GE_128-NEXT: mls v0.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: stp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: mov z1.h, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_256-NEXT: mov z1.h, p1/m, z2.h
+; VBITS_GE_256-NEXT: uunpklo z2.s, z0.h
+; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: uunpklo z3.s, z1.h
+; VBITS_GE_256-NEXT: movprfx z4, z1
+; VBITS_GE_256-NEXT: ext z4.b, z4.b, z1.b, #16
+; VBITS_GE_256-NEXT: uunpklo z4.s, z4.h
+; VBITS_GE_256-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT: movprfx z3, z0
+; VBITS_GE_256-NEXT: ext z3.b, z3.b, z0.b, #16
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: udiv z3.s, p1/m, z3.s, z4.s
+; VBITS_GE_256-NEXT: ptrue p1.h, vl8
+; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h
+; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v16i16:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.h, vl16
+; VBITS_GE_512-NEXT: mov z2.h, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h
+; VBITS_GE_512-NEXT: sel z1.h, p1, z1.h, z2.h
+; VBITS_GE_512-NEXT: ptrue p1.s, vl16
+; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h
+; VBITS_GE_512-NEXT: udivr z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i16>, ptr %a
+ %op2 = load <16 x i16>, ptr %b
+ %mask = icmp ne <16 x i16> %op1, zeroinitializer
+ %res = call <16 x i16> @llvm.masked.urem(<16 x i16> %op1, <16 x i16> %op2, <16 x i1> %mask)
+ store <16 x i16> %res, ptr %a
+ ret void
+}
+
+define void @urem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: urem_v32i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl32
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: uunpklo z3.s, z0.h
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: ptrue p1.s, vl32
+; CHECK-NEXT: uunpklo z2.s, z1.h
+; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i16>, ptr %a
+ %op2 = load <32 x i16>, ptr %b
+ %mask = icmp ne <32 x i16> %op1, zeroinitializer
+ %res = call <32 x i16> @llvm.masked.urem(<32 x i16> %op1, <32 x i16> %op2, <32 x i1> %mask)
+ store <32 x i16> %res, ptr %a
+ ret void
+}
+
+define void @urem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v64i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl64
+; CHECK-NEXT: mov z2.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: uunpklo z3.s, z0.h
+; CHECK-NEXT: sel z1.h, p1, z1.h, z2.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z2.s, z1.h
+; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i16>, ptr %a
+ %op2 = load <64 x i16>, ptr %b
+ %mask = icmp ne <64 x i16> %op1, zeroinitializer
+ %res = call <64 x i16> @llvm.masked.urem(<64 x i16> %op1, <64 x i16> %op2, <64 x i1> %mask)
+ store <64 x i16> %res, ptr %a
+ ret void
+}
+
+define void @urem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v128i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl128
+; CHECK-NEXT: mov z1.h, #1 // =0x1
+; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: mov z1.h, p1/m, z2.h
+; CHECK-NEXT: uunpklo z2.s, z0.h
+; CHECK-NEXT: ptrue p1.s, vl64
+; CHECK-NEXT: uunpklo z3.s, z1.h
+; CHECK-NEXT: movprfx z4, z1
+; CHECK-NEXT: ext z4.b, z4.b, z1.b, #128
+; CHECK-NEXT: uunpklo z4.s, z4.h
+; CHECK-NEXT: udiv z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: ext z3.b, z3.b, z0.b, #128
+; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: udiv z3.s, p1/m, z3.s, z4.s
+; CHECK-NEXT: ptrue p1.h, vl64
+; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h
+; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <128 x i16>, ptr %a
+ %op2 = load <128 x i16>, ptr %b
+ %mask = icmp ne <128 x i16> %op1, zeroinitializer
+ %res = call <128 x i16> @llvm.masked.urem(<128 x i16> %op1, <128 x i16> %op2, <128 x i1> %mask)
+ store <128 x i16> %res, ptr %a
+ ret void
+}
+
+; Vector v2i32 urem are not legal for NEON so use SVE when available.
+define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: urem_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: cmeq v2.2s, v0.2s, #0
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: bic v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: sub v1.2s, v1.2s, v2.2s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i32> %op1, zeroinitializer
+ %res = call <2 x i32> @llvm.masked.urem(<2 x i32> %op1, <2 x i32> %op2, <2 x i1> %mask)
+ ret <2 x i32> %res
+}
+
+; Vector v4i32 urem are not legal for NEON so use SVE when available.
+define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: urem_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmeq v2.4s, v0.4s, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: bic v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: sub v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <4 x i32> %op1, zeroinitializer
+ %res = call <4 x i32> @llvm.masked.urem(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
+ ret <4 x i32> %res
+}
+
+define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: urem_v8i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <8 x i32>, ptr %a
+ %op2 = load <8 x i32>, ptr %b
+ %mask = icmp ne <8 x i32> %op1, zeroinitializer
+ %res = call <8 x i32> @llvm.masked.urem(<8 x i32> %op1, <8 x i32> %op2, <8 x i1> %mask)
+ store <8 x i32> %res, ptr %a
+ ret void
+}
+
+define void @urem_v16i32(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: urem_v16i32:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ptrue p0.s, vl4
+; VBITS_GE_128-NEXT: ldp q16, q17, [x1]
+; VBITS_GE_128-NEXT: ldp q19, q18, [x1, #32]
+; VBITS_GE_128-NEXT: cmeq v2.4s, v1.4s, #0
+; VBITS_GE_128-NEXT: cmeq v3.4s, v0.4s, #0
+; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v2.16b
+; VBITS_GE_128-NEXT: ldp q2, q3, [x0, #32]
+; VBITS_GE_128-NEXT: cmeq v5.4s, v3.4s, #0
+; VBITS_GE_128-NEXT: cmeq v6.4s, v2.4s, #0
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h
+; VBITS_GE_128-NEXT: zip1 v6.8b, v4.8b, v0.8b
+; VBITS_GE_128-NEXT: zip2 v4.8b, v4.8b, v0.8b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: ushll v6.4s, v6.4h, #0
+; VBITS_GE_128-NEXT: ushll v4.4s, v4.4h, #0
+; VBITS_GE_128-NEXT: xtn v5.8b, v5.8h
+; VBITS_GE_128-NEXT: shl v6.4s, v6.4s, #31
+; VBITS_GE_128-NEXT: shl v4.4s, v4.4s, #31
+; VBITS_GE_128-NEXT: zip2 v7.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: zip1 v5.8b, v5.8b, v0.8b
+; VBITS_GE_128-NEXT: cmlt v6.4s, v6.4s, #0
+; VBITS_GE_128-NEXT: cmlt v4.4s, v4.4s, #0
+; VBITS_GE_128-NEXT: and v16.16b, v16.16b, v6.16b
+; VBITS_GE_128-NEXT: mvn v6.16b, v6.16b
+; VBITS_GE_128-NEXT: ushll v7.4s, v7.4h, #0
+; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0
+; VBITS_GE_128-NEXT: and v17.16b, v17.16b, v4.16b
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: sub v6.4s, v16.4s, v6.4s
+; VBITS_GE_128-NEXT: shl v7.4s, v7.4s, #31
+; VBITS_GE_128-NEXT: shl v5.4s, v5.4s, #31
+; VBITS_GE_128-NEXT: sub v4.4s, v17.4s, v4.4s
+; VBITS_GE_128-NEXT: movprfx z16, z0
+; VBITS_GE_128-NEXT: udiv z16.s, p0/m, z16.s, z6.s
+; VBITS_GE_128-NEXT: cmlt v7.4s, v7.4s, #0
+; VBITS_GE_128-NEXT: cmlt v5.4s, v5.4s, #0
+; VBITS_GE_128-NEXT: movprfx z17, z1
+; VBITS_GE_128-NEXT: udiv z17.s, p0/m, z17.s, z4.s
+; VBITS_GE_128-NEXT: and v18.16b, v18.16b, v7.16b
+; VBITS_GE_128-NEXT: mvn v7.16b, v7.16b
+; VBITS_GE_128-NEXT: and v19.16b, v19.16b, v5.16b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v7.4s, v18.4s, v7.4s
+; VBITS_GE_128-NEXT: sub v5.4s, v19.4s, v5.4s
+; VBITS_GE_128-NEXT: movprfx z18, z3
+; VBITS_GE_128-NEXT: udiv z18.s, p0/m, z18.s, z7.s
+; VBITS_GE_128-NEXT: movprfx z19, z2
+; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z5.s
+; VBITS_GE_128-NEXT: mls v0.4s, v16.4s, v6.4s
+; VBITS_GE_128-NEXT: mls v1.4s, v17.4s, v4.4s
+; VBITS_GE_128-NEXT: stp q0, q1, [x0]
+; VBITS_GE_128-NEXT: mls v3.4s, v18.4s, v7.4s
+; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v5.4s
+; VBITS_GE_128-NEXT: stp q2, q3, [x0, #32]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v16i32:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256-NEXT: mov z1.s, #1 // =0x1
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1w { z5.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_256-NEXT: sel z2.s, p1, z2.s, z1.s
+; VBITS_GE_256-NEXT: cmpne p1.s, p0/z, z4.s, #0
+; VBITS_GE_256-NEXT: movprfx z3, z0
+; VBITS_GE_256-NEXT: udiv z3.s, p0/m, z3.s, z2.s
+; VBITS_GE_256-NEXT: mov z1.s, p1/m, z5.s
+; VBITS_GE_256-NEXT: movprfx z5, z4
+; VBITS_GE_256-NEXT: udiv z5.s, p0/m, z5.s, z1.s
+; VBITS_GE_256-NEXT: mls z0.s, p0/m, z3.s, z2.s
+; VBITS_GE_256-NEXT: msb z1.s, p0/m, z5.s, z4.s
+; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v16i32:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: mov z2.s, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; VBITS_GE_512-NEXT: sel z1.s, p1, z1.s, z2.s
+; VBITS_GE_512-NEXT: movprfx z2, z0
+; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <16 x i32>, ptr %a
+ %op2 = load <16 x i32>, ptr %b
+ %mask = icmp ne <16 x i32> %op1, zeroinitializer
+ %res = call <16 x i32> @llvm.masked.urem(<16 x i32> %op1, <16 x i32> %op2, <16 x i1> %mask)
+ store <16 x i32> %res, ptr %a
+ ret void
+}
+
+define void @urem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: urem_v32i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl32
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i32>, ptr %a
+ %op2 = load <32 x i32>, ptr %b
+ %mask = icmp ne <32 x i32> %op1, zeroinitializer
+ %res = call <32 x i32> @llvm.masked.urem(<32 x i32> %op1, <32 x i32> %op2, <32 x i1> %mask)
+ store <32 x i32> %res, ptr %a
+ ret void
+}
+
+define void @urem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v64i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.s, vl64
+; CHECK-NEXT: mov z2.s, #1 // =0x1
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: sel z1.s, p1, z1.s, z2.s
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s
+; CHECK-NEXT: st1w { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <64 x i32>, ptr %a
+ %op2 = load <64 x i32>, ptr %b
+ %mask = icmp ne <64 x i32> %op1, zeroinitializer
+ %res = call <64 x i32> @llvm.masked.urem(<64 x i32> %op1, <64 x i32> %op2, <64 x i1> %mask)
+ store <64 x i32> %res, ptr %a
+ ret void
+}
+
+; Vector i64 urem are not legal for NEON so use SVE when available.
+define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: urem_v1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: fmov x9, d1
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csinc x9, x9, xzr, ne
+; CHECK-NEXT: udiv x10, x8, x9
+; CHECK-NEXT: msub x8, x10, x9, x8
+; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: ret
+ %mask = icmp ne <1 x i64> %op1, zeroinitializer
+ %res = call <1 x i64> @llvm.masked.urem(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask)
+ ret <1 x i64> %res
+}
+
+; Vector i64 urem are not legal for NEON so use SVE when available.
+define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {
+; CHECK-LABEL: urem_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmeq v2.2d, v0.2d, #0
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: bic v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: sub v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %mask = icmp ne <2 x i64> %op1, zeroinitializer
+ %res = call <2 x i64> @llvm.masked.urem(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
+ ret <2 x i64> %res
+}
+
+define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-LABEL: urem_v4i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <4 x i64>, ptr %a
+ %op2 = load <4 x i64>, ptr %b
+ %mask = icmp ne <4 x i64> %op1, zeroinitializer
+ %res = call <4 x i64> @llvm.masked.urem(<4 x i64> %op1, <4 x i64> %op2, <4 x i1> %mask)
+ store <4 x i64> %res, ptr %a
+ ret void
+}
+
+define void @urem_v8i64(ptr %a, ptr %b) #0 {
+; VBITS_GE_128-LABEL: urem_v8i64:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0, #32]
+; VBITS_GE_128-NEXT: ptrue p0.d, vl2
+; VBITS_GE_128-NEXT: ldp q2, q3, [x0]
+; VBITS_GE_128-NEXT: ldp q19, q17, [x1, #32]
+; VBITS_GE_128-NEXT: cmeq v4.2d, v0.2d, #0
+; VBITS_GE_128-NEXT: cmeq v5.2d, v1.2d, #0
+; VBITS_GE_128-NEXT: cmeq v6.2d, v3.2d, #0
+; VBITS_GE_128-NEXT: cmeq v7.2d, v2.2d, #0
+; VBITS_GE_128-NEXT: uzp1 v4.4s, v5.4s, v4.4s
+; VBITS_GE_128-NEXT: uzp1 v5.4s, v7.4s, v6.4s
+; VBITS_GE_128-NEXT: ldp q6, q7, [x1]
+; VBITS_GE_128-NEXT: uzp1 v4.8h, v5.8h, v4.8h
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: xtn v4.8b, v4.8h
+; VBITS_GE_128-NEXT: mov b5, v4.b[0]
+; VBITS_GE_128-NEXT: mov v5.b[4], v4.b[1]
+; VBITS_GE_128-NEXT: ushll v5.2d, v5.2s, #0
+; VBITS_GE_128-NEXT: shl v5.2d, v5.2d, #63
+; VBITS_GE_128-NEXT: cmlt v5.2d, v5.2d, #0
+; VBITS_GE_128-NEXT: and v6.16b, v6.16b, v5.16b
+; VBITS_GE_128-NEXT: mvn v5.16b, v5.16b
+; VBITS_GE_128-NEXT: sub v5.2d, v6.2d, v5.2d
+; VBITS_GE_128-NEXT: mov b6, v4.b[6]
+; VBITS_GE_128-NEXT: movprfx z16, z2
+; VBITS_GE_128-NEXT: udiv z16.d, p0/m, z16.d, z5.d
+; VBITS_GE_128-NEXT: mov v6.b[4], v4.b[7]
+; VBITS_GE_128-NEXT: ushll v6.2d, v6.2s, #0
+; VBITS_GE_128-NEXT: shl v6.2d, v6.2d, #63
+; VBITS_GE_128-NEXT: cmlt v6.2d, v6.2d, #0
+; VBITS_GE_128-NEXT: and v17.16b, v17.16b, v6.16b
+; VBITS_GE_128-NEXT: mvn v6.16b, v6.16b
+; VBITS_GE_128-NEXT: sub v6.2d, v17.2d, v6.2d
+; VBITS_GE_128-NEXT: mov b17, v4.b[4]
+; VBITS_GE_128-NEXT: movprfx z18, z0
+; VBITS_GE_128-NEXT: udiv z18.d, p0/m, z18.d, z6.d
+; VBITS_GE_128-NEXT: mov v17.b[4], v4.b[5]
+; VBITS_GE_128-NEXT: ushll v17.2d, v17.2s, #0
+; VBITS_GE_128-NEXT: mls z2.d, p0/m, z16.d, z5.d
+; VBITS_GE_128-NEXT: shl v17.2d, v17.2d, #63
+; VBITS_GE_128-NEXT: cmlt v17.2d, v17.2d, #0
+; VBITS_GE_128-NEXT: and v19.16b, v19.16b, v17.16b
+; VBITS_GE_128-NEXT: mvn v17.16b, v17.16b
+; VBITS_GE_128-NEXT: sub v17.2d, v19.2d, v17.2d
+; VBITS_GE_128-NEXT: mov b19, v4.b[2]
+; VBITS_GE_128-NEXT: movprfx z20, z1
+; VBITS_GE_128-NEXT: udiv z20.d, p0/m, z20.d, z17.d
+; VBITS_GE_128-NEXT: mov v19.b[4], v4.b[3]
+; VBITS_GE_128-NEXT: ushll v4.2d, v19.2s, #0
+; VBITS_GE_128-NEXT: mls z0.d, p0/m, z18.d, z6.d
+; VBITS_GE_128-NEXT: shl v4.2d, v4.2d, #63
+; VBITS_GE_128-NEXT: cmlt v4.2d, v4.2d, #0
+; VBITS_GE_128-NEXT: and v7.16b, v7.16b, v4.16b
+; VBITS_GE_128-NEXT: mvn v4.16b, v4.16b
+; VBITS_GE_128-NEXT: sub v4.2d, v7.2d, v4.2d
+; VBITS_GE_128-NEXT: movprfx z7, z3
+; VBITS_GE_128-NEXT: udiv z7.d, p0/m, z7.d, z4.d
+; VBITS_GE_128-NEXT: mls z1.d, p0/m, z20.d, z17.d
+; VBITS_GE_128-NEXT: stp q1, q0, [x0, #32]
+; VBITS_GE_128-NEXT: mls z3.d, p0/m, z7.d, z4.d
+; VBITS_GE_128-NEXT: stp q2, q3, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: urem_v8i64:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_256-NEXT: cmpne p2.d, p0/z, z1.d, #0
+; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: mov z3.d, p2/z, #-1 // =0xffffffffffffffff
+; VBITS_GE_256-NEXT: ptrue p1.s, vl4
+; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
+; VBITS_GE_256-NEXT: ptrue p1.d
+; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
+; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
+; VBITS_GE_256-NEXT: zip2 v3.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: zip1 v2.8b, v2.8b, v0.8b
+; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h
+; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h
+; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s
+; VBITS_GE_256-NEXT: uunpklo z2.d, z2.s
+; VBITS_GE_256-NEXT: lsl z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: lsl z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: asr z3.d, z3.d, #63
+; VBITS_GE_256-NEXT: asr z2.d, z2.d, #63
+; VBITS_GE_256-NEXT: and z3.d, z3.d, #0x1
+; VBITS_GE_256-NEXT: and z2.d, z2.d, #0x1
+; VBITS_GE_256-NEXT: cmpne p2.d, p1/z, z3.d, #0
+; VBITS_GE_256-NEXT: mov z3.d, #1 // =0x1
+; VBITS_GE_256-NEXT: sel z4.d, p2, z4.d, z3.d
+; VBITS_GE_256-NEXT: cmpne p2.d, p1/z, z2.d, #0
+; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: movprfx z5, z0
+; VBITS_GE_256-NEXT: udiv z5.d, p0/m, z5.d, z4.d
+; VBITS_GE_256-NEXT: sel z2.d, p2, z2.d, z3.d
+; VBITS_GE_256-NEXT: movprfx z3, z1
+; VBITS_GE_256-NEXT: udiv z3.d, p0/m, z3.d, z2.d
+; VBITS_GE_256-NEXT: mls z0.d, p0/m, z5.d, z4.d
+; VBITS_GE_256-NEXT: mls z1.d, p0/m, z3.d, z2.d
+; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; VBITS_GE_512-LABEL: urem_v8i64:
+; VBITS_GE_512: // %bb.0:
+; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: mov z2.d, #1 // =0x1
+; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; VBITS_GE_512-NEXT: sel z1.d, p1, z1.d, z2.d
+; VBITS_GE_512-NEXT: movprfx z2, z0
+; VBITS_GE_512-NEXT: udiv z2.d, p0/m, z2.d, z1.d
+; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
+; VBITS_GE_512-NEXT: ret
+ %op1 = load <8 x i64>, ptr %a
+ %op2 = load <8 x i64>, ptr %b
+ %mask = icmp ne <8 x i64> %op1, zeroinitializer
+ %res = call <8 x i64> @llvm.masked.urem(<8 x i64> %op1, <8 x i64> %op2, <8 x i1> %mask)
+ store <8 x i64> %res, ptr %a
+ ret void
+}
+
+define void @urem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-LABEL: urem_v16i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <16 x i64>, ptr %a
+ %op2 = load <16 x i64>, ptr %b
+ %mask = icmp ne <16 x i64> %op1, zeroinitializer
+ %res = call <16 x i64> @llvm.masked.urem(<16 x i64> %op1, <16 x i64> %op2, <16 x i1> %mask)
+ store <16 x i64> %res, ptr %a
+ ret void
+}
+
+define void @urem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-LABEL: urem_v32i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: mov z2.d, #1 // =0x1
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: sel z1.d, p1, z1.d, z2.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %op1 = load <32 x i64>, ptr %a
+ %op2 = load <32 x i64>, ptr %b
+ %mask = icmp ne <32 x i64> %op1, zeroinitializer
+ %res = call <32 x i64> @llvm.masked.urem(<32 x i64> %op1, <32 x i64> %op2, <32 x i1> %mask)
+ store <32 x i64> %res, ptr %a
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
More information about the llvm-commits
mailing list