[llvm] a23f07f - [RISCV] Add merge operands to more RISCVISD::*_VL opcodes.
Craig Topper via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 30 10:33:36 PDT 2022
Author: Craig Topper
Date: 2022-07-30T10:26:38-07:00
New Revision: a23f07fb1daad8a1fc9108c5de8c6a1503fda06c
URL: https://github.com/llvm/llvm-project/commit/a23f07fb1daad8a1fc9108c5de8c6a1503fda06c
DIFF: https://github.com/llvm/llvm-project/commit/a23f07fb1daad8a1fc9108c5de8c6a1503fda06c.diff
LOG: [RISCV] Add merge operands to more RISCVISD::*_VL opcodes.
This adds a merge operand to all of the binary _VL nodes. Including
integer and widening. They all share multiclasses in tablegen
so doing them all at once was easiest.
I plan to use FADD_VL in an upcoming patch. The rest are just for
consistency to keep tablegen working.
This does reduce the isel table size by about 25k so that's nice.
Reviewed By: reames
Differential Revision: https://reviews.llvm.org/D130816
Added:
Modified:
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.h
llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 9f28f73de158..f9119884613f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1929,8 +1929,8 @@ static SDValue lowerFROUND(SDValue Op, SelectionDAG &DAG,
DAG.getUNDEF(ContainerVT), SplatVal, VL);
// Add the adjustment.
- SDValue Adjust =
- DAG.getNode(RISCVISD::FADD_VL, DL, ContainerVT, Abs, Splat, Mask, VL);
+ SDValue Adjust = DAG.getNode(RISCVISD::FADD_VL, DL, ContainerVT, Abs, Splat,
+ DAG.getUNDEF(ContainerVT), Mask, VL);
// Truncate to integer and convert back to fp.
MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
@@ -2798,19 +2798,21 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
TrueMask = getAllOnesMask(HalfContainerVT, VL, DL, DAG);
// Widen V1 and V2 with 0s and add one copy of V2 to V1.
- SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
- V2, TrueMask, VL);
+ SDValue Add =
+ DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1, V2,
+ DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
DAG.getUNDEF(IntHalfVT),
DAG.getAllOnesConstant(DL, XLenVT));
- SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
- V2, Multiplier, TrueMask, VL);
+ SDValue WidenMul =
+ DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT, V2, Multiplier,
+ DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Add the new copies to our previous addition giving us 2^eltbits copies of
// V2. This is equivalent to shifting V2 left by eltbits. This should
// combine with the vwmulu.vv above to form vwmaccu.vv.
Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
- TrueMask, VL);
+ DAG.getUNDEF(WideIntContainerVT), TrueMask, VL);
// Cast back to ContainerVT. We need to re-create a new ContainerVT in case
// WideIntContainerVT is a larger fractional LMUL than implied by the fixed
// vector VT.
@@ -3534,15 +3536,15 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::SETCC:
return lowerFixedLengthVectorSetccToRVV(Op, DAG);
case ISD::ADD:
- return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL, /*HasMergeOp*/ true);
case ISD::SUB:
- return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL, /*HasMergeOp*/ true);
case ISD::MUL:
- return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL, /*HasMergeOp*/ true);
case ISD::MULHS:
- return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL, /*HasMergeOp*/ true);
case ISD::MULHU:
- return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL, /*HasMergeOp*/ true);
case ISD::AND:
return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
RISCVISD::AND_VL);
@@ -3553,13 +3555,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
RISCVISD::XOR_VL);
case ISD::SDIV:
- return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL, /*HasMergeOp*/ true);
case ISD::SREM:
- return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL, /*HasMergeOp*/ true);
case ISD::UDIV:
- return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL, /*HasMergeOp*/ true);
case ISD::UREM:
- return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL, /*HasMergeOp*/ true);
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
@@ -3570,21 +3572,25 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
"Unexpected custom legalisation");
return SDValue();
case ISD::SADDSAT:
- return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL,
+ /*HasMergeOp*/ true);
case ISD::UADDSAT:
- return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL,
+ /*HasMergeOp*/ true);
case ISD::SSUBSAT:
- return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL,
+ /*HasMergeOp*/ true);
case ISD::USUBSAT:
- return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL,
+ /*HasMergeOp*/ true);
case ISD::FADD:
- return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL, /*HasMergeOp*/ true);
case ISD::FSUB:
- return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL, /*HasMergeOp*/ true);
case ISD::FMUL:
- return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL, /*HasMergeOp*/ true);
case ISD::FDIV:
- return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL, /*HasMergeOp*/ true);
case ISD::FNEG:
return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
case ISD::FABS:
@@ -3594,17 +3600,19 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::FMA:
return lowerToScalableOp(Op, DAG, RISCVISD::VFMADD_VL);
case ISD::SMIN:
- return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL, /*HasMergeOp*/ true);
case ISD::SMAX:
- return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL, /*HasMergeOp*/ true);
case ISD::UMIN:
- return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL, /*HasMergeOp*/ true);
case ISD::UMAX:
- return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL, /*HasMergeOp*/ true);
case ISD::FMINNUM:
- return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL,
+ /*HasMergeOp*/ true);
case ISD::FMAXNUM:
- return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
+ return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL,
+ /*HasMergeOp*/ true);
case ISD::ABS:
return lowerABS(Op, DAG);
case ISD::CTLZ_ZERO_UNDEF:
@@ -3631,19 +3639,19 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::VP_MERGE:
return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
case ISD::VP_ADD:
- return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::ADD_VL, /*HasMergeOp*/ true);
case ISD::VP_SUB:
- return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SUB_VL, /*HasMergeOp*/ true);
case ISD::VP_MUL:
- return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::MUL_VL, /*HasMergeOp*/ true);
case ISD::VP_SDIV:
- return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_UDIV:
- return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_SREM:
- return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SREM_VL, /*HasMergeOp*/ true);
case ISD::VP_UREM:
- return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::UREM_VL, /*HasMergeOp*/ true);
case ISD::VP_AND:
return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
case ISD::VP_OR:
@@ -3651,19 +3659,19 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::VP_XOR:
return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
case ISD::VP_ASHR:
- return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SRA_VL, /*HasMergeOp*/ true);
case ISD::VP_LSHR:
- return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SRL_VL, /*HasMergeOp*/ true);
case ISD::VP_SHL:
- return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::SHL_VL, /*HasMergeOp*/ true);
case ISD::VP_FADD:
- return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::FADD_VL, /*HasMergeOp*/ true);
case ISD::VP_FSUB:
- return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL, /*HasMergeOp*/ true);
case ISD::VP_FMUL:
- return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL, /*HasMergeOp*/ true);
case ISD::VP_FDIV:
- return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
+ return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL, /*HasMergeOp*/ true);
case ISD::VP_FNEG:
return lowerVPOp(Op, DAG, RISCVISD::FNEG_VL);
case ISD::VP_FMA:
@@ -4343,8 +4351,8 @@ SDValue RISCVTargetLowering::lowerVectorMaskTruncLike(SDValue Op,
DAG.getUNDEF(ContainerVT), SplatZero, VL);
MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
- SDValue Trunc =
- DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
+ SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
+ DAG.getUNDEF(ContainerVT), Mask, VL);
Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
DAG.getCondCode(ISD::SETNE), Mask, VL);
if (MaskVT.isFixedLengthVector())
@@ -5807,8 +5815,8 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
VLMinus1, DAG.getRegister(RISCV::X0, XLenVT));
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
- SDValue Indices =
- DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
+ SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
+ DAG.getUNDEF(IntVT), Mask, VL);
return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
DAG.getUNDEF(VecVT), Mask, VL);
@@ -6071,9 +6079,10 @@ SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
MVT VT = Op.getSimpleValueType();
if (VT.getVectorElementType() == MVT::i1)
- return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
+ return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMergeOp*/ false,
+ /*HasMask*/ false);
- return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
+ return lowerToScalableOp(Op, DAG, VecOpc, /*HasMergeOp*/ true);
}
SDValue
@@ -6087,7 +6096,7 @@ RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
}
- return lowerToScalableOp(Op, DAG, Opc);
+ return lowerToScalableOp(Op, DAG, Opc, /*HasMergeOp*/ true);
}
// Lower vector ABS to smax(X, sub(0, X)).
@@ -6107,10 +6116,10 @@ SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
SDValue SplatZero = DAG.getNode(
RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
DAG.getConstant(0, DL, Subtarget.getXLenVT()));
- SDValue NegX =
- DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
- SDValue Max =
- DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
+ SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
+ DAG.getUNDEF(ContainerVT), Mask, VL);
+ SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
+ DAG.getUNDEF(ContainerVT), Mask, VL);
return convertFromScalableVector(VT, Max, DAG, Subtarget);
}
@@ -6163,7 +6172,7 @@ SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
}
SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
- unsigned NewOpc,
+ unsigned NewOpc, bool HasMergeOp,
bool HasMask) const {
MVT VT = Op.getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(VT);
@@ -6188,6 +6197,8 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+ if (HasMergeOp)
+ Ops.push_back(DAG.getUNDEF(ContainerVT));
if (HasMask)
Ops.push_back(Mask);
Ops.push_back(VL);
@@ -6202,14 +6213,22 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
// * Fixed-length vectors are converted to their scalable-vector container
// types.
SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
- unsigned RISCVISDOpc) const {
+ unsigned RISCVISDOpc,
+ bool HasMergeOp) const {
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
SmallVector<SDValue, 4> Ops;
+ MVT ContainerVT = VT;
+ if (VT.isFixedLengthVector())
+ ContainerVT = getContainerForFixedLengthVector(VT);
+
for (const auto &OpIdx : enumerate(Op->ops())) {
SDValue V = OpIdx.value();
assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
+ // Add dummy merge value before the mask.
+ if (HasMergeOp && *ISD::getVPMaskIdx(Op.getOpcode()) == OpIdx.index())
+ Ops.push_back(DAG.getUNDEF(ContainerVT));
// Pass through operands which aren't fixed-length vectors.
if (!V.getValueType().isFixedLengthVector()) {
Ops.push_back(V);
@@ -6226,8 +6245,6 @@ SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
if (!VT.isFixedLengthVector())
return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
- MVT ContainerVT = getContainerForFixedLengthVector(VT);
-
SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
@@ -6483,7 +6500,7 @@ SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
unsigned VecOpc) const {
MVT VT = Op.getSimpleValueType();
if (VT.getVectorElementType() != MVT::i1)
- return lowerVPOp(Op, DAG, VecOpc);
+ return lowerVPOp(Op, DAG, VecOpc, true);
// It is safe to drop mask parameter as masked-off elements are undef.
SDValue Op1 = Op->getOperand(0);
@@ -7281,8 +7298,9 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
DAG.getUNDEF(ContainerVT),
DAG.getConstant(32, DL, XLenVT), VL);
- SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
- ThirtyTwoV, Mask, VL);
+ SDValue LShr32 =
+ DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
+ DAG.getUNDEF(ContainerVT), Mask, VL);
SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
@@ -7389,8 +7407,8 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue ThirtyTwoV =
DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
DAG.getConstant(32, DL, XLenVT), VL);
- SDValue LShr32 =
- DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
+ SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
+ DAG.getUNDEF(VecVT), Mask, VL);
SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
Results.push_back(
@@ -8298,8 +8316,9 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
VT.getVectorElementCount());
- SDValue Mask = N->getOperand(2);
- SDValue VL = N->getOperand(3);
+ SDValue Merge = N->getOperand(2);
+ SDValue Mask = N->getOperand(3);
+ SDValue VL = N->getOperand(4);
SDLoc DL(N);
@@ -8319,7 +8338,7 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
else
WOpc = IsAdd ? RISCVISD::VWADDU_W_VL : RISCVISD::VWSUBU_W_VL;
- return DAG.getNode(WOpc, DL, VT, Op0, Op1, Mask, VL);
+ return DAG.getNode(WOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}
// FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
@@ -8333,8 +8352,9 @@ static SDValue combineADDSUB_VLToVWADDSUB_VL(SDNode *N, SelectionDAG &DAG,
static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);
- SDValue Mask = N->getOperand(2);
- SDValue VL = N->getOperand(3);
+ SDValue Merge = N->getOperand(2);
+ SDValue Mask = N->getOperand(3);
+ SDValue VL = N->getOperand(4);
MVT VT = N->getSimpleValueType(0);
MVT NarrowVT = Op1.getSimpleValueType();
@@ -8364,7 +8384,7 @@ static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
// Re-introduce narrower extends if needed.
if (Op0.getValueType() != NarrowVT)
Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
- return DAG.getNode(VOpc, DL, VT, Op0, Op1, Mask, VL);
+ return DAG.getNode(VOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}
bool IsAdd = N->getOpcode() == RISCVISD::VWADD_W_VL ||
@@ -8396,7 +8416,7 @@ static SDValue combineVWADD_W_VL_VWSUB_W_VL(SDNode *N, SelectionDAG &DAG) {
Op0 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
DAG.getUNDEF(NarrowVT), Op0, VL);
- return DAG.getNode(VOpc, DL, VT, Op1, Op0, Mask, VL);
+ return DAG.getNode(VOpc, DL, VT, Op1, Op0, Merge, Mask, VL);
}
return SDValue();
@@ -8418,8 +8438,9 @@ static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
return SDValue();
- SDValue Mask = N->getOperand(2);
- SDValue VL = N->getOperand(3);
+ SDValue Merge = N->getOperand(2);
+ SDValue Mask = N->getOperand(3);
+ SDValue VL = N->getOperand(4);
// Make sure the mask and VL match.
if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
@@ -8497,7 +8518,7 @@ static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
unsigned WMulOpc = RISCVISD::VWMULSU_VL;
if (!IsVWMULSU)
WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
- return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
+ return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Merge, Mask, VL);
}
static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
@@ -9230,7 +9251,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
ShAmt.getOperand(1), VL);
return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
- N->getOperand(2), N->getOperand(3));
+ N->getOperand(2), N->getOperand(3), N->getOperand(4));
}
break;
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index c9943fd11ec7..19d8b3647452 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -213,9 +213,8 @@ enum NodeType : unsigned {
VECREDUCE_FMIN_VL,
VECREDUCE_FMAX_VL,
- // Vector binary and unary ops with a mask as a third operand, and VL as a
- // fourth operand.
- // FIXME: Can we replace these with ISD::VP_*?
+ // Vector binary ops with a merge as a third operand, a mask as a fourth
+ // operand, and VL as a fifth operand.
ADD_VL,
AND_VL,
MUL_VL,
@@ -229,32 +228,30 @@ enum NodeType : unsigned {
UDIV_VL,
UREM_VL,
XOR_VL,
+ SMIN_VL,
+ SMAX_VL,
+ UMIN_VL,
+ UMAX_VL,
SADDSAT_VL,
UADDSAT_VL,
SSUBSAT_VL,
USUBSAT_VL,
+ MULHS_VL,
+ MULHU_VL,
FADD_VL,
FSUB_VL,
FMUL_VL,
FDIV_VL,
+ FMINNUM_VL,
+ FMAXNUM_VL,
+
+ // Vector unary ops with a mask as a second operand and VL as a third operand.
FNEG_VL,
FABS_VL,
FSQRT_VL,
- VFMADD_VL,
- VFNMADD_VL,
- VFMSUB_VL,
- VFNMSUB_VL,
FCOPYSIGN_VL,
- SMIN_VL,
- SMAX_VL,
- UMIN_VL,
- UMAX_VL,
- FMINNUM_VL,
- FMAXNUM_VL,
- MULHS_VL,
- MULHU_VL,
FP_TO_SINT_VL,
FP_TO_UINT_VL,
SINT_TO_FP_VL,
@@ -262,7 +259,14 @@ enum NodeType : unsigned {
FP_ROUND_VL,
FP_EXTEND_VL,
- // Widening instructions
+ // Vector FMA ops with a mask as a fourth operand and VL as a fifth operand.
+ VFMADD_VL,
+ VFNMADD_VL,
+ VFMSUB_VL,
+ VFNMSUB_VL,
+
+ // Widening instructions with a merge value a third operand, a mask as a
+ // fourth operand, and VL as a fifth operand.
VWMUL_VL,
VWMULU_VL,
VWMULSU_VL,
@@ -679,8 +683,9 @@ class RISCVTargetLowering : public TargetLowering {
SDValue lowerFixedLengthVectorSelectToRVV(SDValue Op,
SelectionDAG &DAG) const;
SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc,
- bool HasMask = true) const;
- SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc) const;
+ bool HasMergeOp = false, bool HasMask = true) const;
+ SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc,
+ bool HasMergeOp = false) const;
SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, unsigned MaskOpc,
unsigned VecOpc) const;
SDValue lowerVPExtMaskOp(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 3c32fd639070..d59cf542491c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -21,24 +21,26 @@
// Helpers to define the VL patterns.
//===----------------------------------------------------------------------===//
-def SDT_RISCVIntBinOp_VL : SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>,
+def SDT_RISCVIntBinOp_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
SDTCisVec<0>, SDTCisInt<0>,
- SDTCVecEltisVT<3, i1>,
- SDTCisSameNumEltsAs<0, 3>,
- SDTCisVT<4, XLenVT>]>;
+ SDTCisSameAs<0, 3>,
+ SDTCVecEltisVT<4, i1>,
+ SDTCisSameNumEltsAs<0, 4>,
+ SDTCisVT<5, XLenVT>]>;
def SDT_RISCVFPUnOp_VL : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>,
SDTCisVec<0>, SDTCisFP<0>,
SDTCVecEltisVT<2, i1>,
SDTCisSameNumEltsAs<0, 2>,
SDTCisVT<3, XLenVT>]>;
-def SDT_RISCVFPBinOp_VL : SDTypeProfile<1, 4, [SDTCisSameAs<0, 1>,
+def SDT_RISCVFPBinOp_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
SDTCisVec<0>, SDTCisFP<0>,
- SDTCVecEltisVT<3, i1>,
- SDTCisSameNumEltsAs<0, 3>,
- SDTCisVT<4, XLenVT>]>;
+ SDTCisSameAs<0, 3>,
+ SDTCVecEltisVT<4, i1>,
+ SDTCisSameNumEltsAs<0, 4>,
+ SDTCisVT<5, XLenVT>]>;
def SDT_RISCVCopySign_VL : SDTypeProfile<1, 5, [SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
@@ -230,12 +232,13 @@ def riscv_trunc_vector_vl : SDNode<"RISCVISD::TRUNCATE_VECTOR_VL",
SDTCVecEltisVT<2, i1>,
SDTCisVT<3, XLenVT>]>>;
-def SDT_RISCVVWBinOp_VL : SDTypeProfile<1, 4, [SDTCisVec<0>,
+def SDT_RISCVVWBinOp_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
SDTCisSameNumEltsAs<0, 1>,
SDTCisSameAs<1, 2>,
- SDTCisSameNumEltsAs<1, 3>,
- SDTCVecEltisVT<3, i1>,
- SDTCisVT<4, XLenVT>]>;
+ SDTCisSameAs<0, 3>,
+ SDTCisSameNumEltsAs<1, 4>,
+ SDTCVecEltisVT<4, i1>,
+ SDTCisVT<5, XLenVT>]>;
def riscv_vwmul_vl : SDNode<"RISCVISD::VWMUL_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwmulu_vl : SDNode<"RISCVISD::VWMULU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwmulsu_vl : SDNode<"RISCVISD::VWMULSU_VL", SDT_RISCVVWBinOp_VL>;
@@ -244,13 +247,14 @@ def riscv_vwaddu_vl : SDNode<"RISCVISD::VWADDU_VL", SDT_RISCVVWBinOp_VL, [SDNPCo
def riscv_vwsub_vl : SDNode<"RISCVISD::VWSUB_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
def riscv_vwsubu_vl : SDNode<"RISCVISD::VWSUBU_VL", SDT_RISCVVWBinOp_VL, [SDNPCommutative]>;
-def SDT_RISCVVWBinOpW_VL : SDTypeProfile<1, 4, [SDTCisVec<0>,
+def SDT_RISCVVWBinOpW_VL : SDTypeProfile<1, 5, [SDTCisVec<0>,
SDTCisSameAs<0, 1>,
SDTCisSameNumEltsAs<1, 2>,
SDTCisOpSmallerThanOp<2, 1>,
- SDTCisSameNumEltsAs<1, 3>,
- SDTCVecEltisVT<3, i1>,
- SDTCisVT<4, XLenVT>]>;
+ SDTCisSameAs<0, 3>,
+ SDTCisSameNumEltsAs<1, 4>,
+ SDTCVecEltisVT<4, i1>,
+ SDTCisVT<5, XLenVT>]>;
def riscv_vwadd_w_vl : SDNode<"RISCVISD::VWADD_W_VL", SDT_RISCVVWBinOpW_VL>;
def riscv_vwaddu_w_vl : SDNode<"RISCVISD::VWADDU_W_VL", SDT_RISCVVWBinOpW_VL>;
def riscv_vwsub_w_vl : SDNode<"RISCVISD::VWSUB_W_VL", SDT_RISCVVWBinOpW_VL>;
@@ -261,27 +265,31 @@ def SDTRVVVecReduce : SDTypeProfile<1, 5, [
SDTCVecEltisVT<4, i1>, SDTCisSameNumEltsAs<2, 4>, SDTCisVT<5, XLenVT>
]>;
-def riscv_mul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
+def riscv_mul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
+ node:$E),
(riscv_mul_vl node:$A, node:$B, node:$C,
- node:$D), [{
+ node:$D, node:$E), [{
return N->hasOneUse();
}]>;
-def riscv_vwmul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
+def riscv_vwmul_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
+ node:$E),
(riscv_vwmul_vl node:$A, node:$B, node:$C,
- node:$D), [{
+ node:$D, node:$E), [{
return N->hasOneUse();
}]>;
-def riscv_vwmulu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
+def riscv_vwmulu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
+ node:$E),
(riscv_vwmulu_vl node:$A, node:$B, node:$C,
- node:$D), [{
+ node:$D, node:$E), [{
return N->hasOneUse();
}]>;
-def riscv_vwmulsu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D),
+def riscv_vwmulsu_vl_oneuse : PatFrag<(ops node:$A, node:$B, node:$C, node:$D,
+ node:$E),
(riscv_vwmulsu_vl node:$A, node:$B, node:$C,
- node:$D), [{
+ node:$D, node:$E), [{
return N->hasOneUse();
}]>;
@@ -331,15 +339,17 @@ multiclass VPatBinaryVL_V<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
+ VReg result_reg_class,
VReg op1_reg_class,
VReg op2_reg_class> {
def : Pat<(result_type (vop
(op1_type op1_reg_class:$rs1),
(op2_type op2_reg_class:$rs2),
+ (result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#suffix#"_"# vlmul.MX#"_MASK")
- (result_type (IMPLICIT_DEF)),
+ result_reg_class:$merge,
op1_reg_class:$rs1,
op2_reg_class:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
@@ -354,16 +364,18 @@ multiclass VPatBinaryVL_XI<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
+ VReg result_reg_class,
VReg vop_reg_class,
ComplexPattern SplatPatKind,
DAGOperand xop_kind> {
def : Pat<(result_type (vop
(vop1_type vop_reg_class:$rs1),
(vop2_type (SplatPatKind (XLenVT xop_kind:$rs2))),
+ (result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#_#suffix#_# vlmul.MX#"_MASK")
- (result_type (IMPLICIT_DEF)),
+ result_reg_class:$merge,
vop_reg_class:$rs1,
xop_kind:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
@@ -373,10 +385,12 @@ multiclass VPatBinaryVL_VV_VX<SDNode vop, string instruction_name> {
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
+ vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
+ vti.RegClass>;
defm : VPatBinaryVL_XI<vop, instruction_name, "VX",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass, SplatPat, GPR>;
+ vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
+ SplatPat, GPR>;
}
}
@@ -386,7 +400,7 @@ multiclass VPatBinaryVL_VV_VX_VI<SDNode vop, string instruction_name,
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_XI<vop, instruction_name, "VI",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass,
+ vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
!cast<ComplexPattern>(SplatPat#_#ImmType),
ImmType>;
}
@@ -398,10 +412,12 @@ multiclass VPatBinaryWVL_VV_VX<SDNode vop, string instruction_name> {
defvar wti = VtiToWti.Wti;
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
wti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
+ vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
+ vti.RegClass>;
defm : VPatBinaryVL_XI<vop, instruction_name, "VX",
wti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass, SplatPat, GPR>;
+ vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
+ SplatPat, GPR>;
}
}
multiclass VPatBinaryWVL_VV_VX_WV_WX<SDNode vop, SDNode vop_w,
@@ -412,10 +428,12 @@ multiclass VPatBinaryWVL_VV_VX_WV_WX<SDNode vop, SDNode vop_w,
defvar wti = VtiToWti.Wti;
defm : VPatBinaryVL_V<vop_w, instruction_name, "WV",
wti.Vector, wti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass>;
+ vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
+ vti.RegClass>;
defm : VPatBinaryVL_XI<vop_w, instruction_name, "WX",
wti.Vector, wti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, wti.RegClass, SplatPat, GPR>;
+ vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
+ SplatPat, GPR>;
}
}
@@ -426,14 +444,16 @@ multiclass VPatBinaryVL_VF<SDNode vop,
ValueType mask_type,
int sew,
LMULInfo vlmul,
+ VReg result_reg_class,
VReg vop_reg_class,
RegisterClass scalar_reg_class> {
def : Pat<(result_type (vop (vop_type vop_reg_class:$rs1),
(vop_type (SplatFPOp scalar_reg_class:$rs2)),
+ (result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#vlmul.MX#"_MASK")
- (result_type (IMPLICIT_DEF)),
+ result_reg_class:$merge,
vop_reg_class:$rs1,
scalar_reg_class:$rs2,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
@@ -443,10 +463,12 @@ multiclass VPatBinaryFPVL_VV_VF<SDNode vop, string instruction_name> {
foreach vti = AllFloatVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
- vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
+ vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass,
+ vti.RegClass>;
defm : VPatBinaryVL_VF<vop, instruction_name#"_V"#vti.ScalarSuffix,
vti.Vector, vti.Vector, vti.Mask, vti.Log2SEW,
- vti.LMul, vti.RegClass, vti.ScalarRegClass>;
+ vti.LMul, vti.RegClass, vti.RegClass,
+ vti.ScalarRegClass>;
}
}
@@ -454,10 +476,11 @@ multiclass VPatBinaryFPVL_R_VF<SDNode vop, string instruction_name> {
foreach fvti = AllFloatVectors in {
def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2),
fvti.RegClass:$rs1,
+ (fvti.Vector fvti.RegClass:$merge),
(fvti.Mask V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK")
- (fvti.Vector (IMPLICIT_DEF)),
+ fvti.RegClass:$merge,
fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2,
(fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>;
}
@@ -786,7 +809,7 @@ multiclass VPatWidenBinaryFPVL_VV_VF<SDNode op, PatFrags extop, string instructi
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Vector (extop (fvti.Vector fvti.RegClass:$rs1),
(fvti.Mask true_mask), VLOpFrag)),
- (fwti.Mask true_mask), VLOpFrag)),
+ srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_"#fvti.LMul.MX)
fvti.RegClass:$rs2, fvti.RegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
@@ -794,7 +817,7 @@ multiclass VPatWidenBinaryFPVL_VV_VF<SDNode op, PatFrags extop, string instructi
(fvti.Mask true_mask), VLOpFrag)),
(fwti.Vector (extop (fvti.Vector (SplatFPOp fvti.ScalarRegClass:$rs1)),
(fvti.Mask true_mask), VLOpFrag)),
- (fwti.Mask true_mask), VLOpFrag)),
+ srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
fvti.RegClass:$rs2, fvti.ScalarRegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
@@ -808,14 +831,14 @@ multiclass VPatWidenBinaryFPVL_WV_WF<SDNode op, PatFrags extop, string instructi
def : Pat<(fwti.Vector (op (fwti.Vector fwti.RegClass:$rs2),
(fwti.Vector (extop (fvti.Vector fvti.RegClass:$rs1),
(fvti.Mask true_mask), VLOpFrag)),
- (fwti.Mask true_mask), VLOpFrag)),
+ srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WV_"#fvti.LMul.MX)
fwti.RegClass:$rs2, fvti.RegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
def : Pat<(fwti.Vector (op (fwti.Vector fwti.RegClass:$rs2),
(fwti.Vector (extop (fvti.Vector (SplatFPOp fvti.ScalarRegClass:$rs1)),
(fvti.Mask true_mask), VLOpFrag)),
- (fwti.Mask true_mask), VLOpFrag)),
+ srcvalue, (fwti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_W"#fvti.ScalarSuffix#"_"#fvti.LMul.MX)
fwti.RegClass:$rs2, fvti.ScalarRegClass:$rs1,
GPR:$vl, fvti.Log2SEW)>;
@@ -837,7 +860,7 @@ multiclass VPatNarrowShiftSplatExt_WX<SDNode op, PatFrags extop, string instruct
(op (wti.Vector wti.RegClass:$rs2),
(wti.Vector (extop (vti.Vector (SplatPat GPR:$rs1)),
(vti.Mask true_mask), VLOpFrag)),
- (wti.Mask true_mask), VLOpFrag),
+ srcvalue, (wti.Mask true_mask), VLOpFrag),
(vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WX_"#vti.LMul.MX)
wti.RegClass:$rs2, GPR:$rs1, GPR:$vl, vti.Log2SEW)>;
@@ -853,8 +876,8 @@ multiclass VPatMultiplyAddVL_VV_VX<SDNode op, string instruction_name> {
(op vti.RegClass:$rs2,
(riscv_mul_vl_oneuse vti.RegClass:$rs1,
vti.RegClass:$rd,
- (vti.Mask true_mask), VLOpFrag),
- (vti.Mask true_mask), VLOpFrag)),
+ srcvalue, (vti.Mask true_mask), VLOpFrag),
+ srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_"# suffix)
vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
@@ -864,8 +887,8 @@ multiclass VPatMultiplyAddVL_VV_VX<SDNode op, string instruction_name> {
(op vti.RegClass:$rs2,
(riscv_mul_vl_oneuse (SplatPat XLenVT:$rs1),
vti.RegClass:$rd,
- (vti.Mask true_mask), VLOpFrag),
- (vti.Mask true_mask), VLOpFrag)),
+ srcvalue, (vti.Mask true_mask), VLOpFrag),
+ srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VX_" # suffix)
vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
@@ -880,8 +903,8 @@ multiclass VPatWidenMultiplyAddVL_VV_VX<PatFrag op1, string instruction_name> {
(riscv_add_vl wti.RegClass:$rd,
(op1 vti.RegClass:$rs1,
(vti.Vector vti.RegClass:$rs2),
- (vti.Mask true_mask), VLOpFrag),
- (vti.Mask true_mask), VLOpFrag)),
+ srcvalue, (vti.Mask true_mask), VLOpFrag),
+ srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VV_" # vti.LMul.MX)
wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
@@ -889,8 +912,8 @@ multiclass VPatWidenMultiplyAddVL_VV_VX<PatFrag op1, string instruction_name> {
(riscv_add_vl wti.RegClass:$rd,
(op1 (SplatPat XLenVT:$rs1),
(vti.Vector vti.RegClass:$rs2),
- (vti.Mask true_mask), VLOpFrag),
- (vti.Mask true_mask), VLOpFrag)),
+ srcvalue, (vti.Mask true_mask), VLOpFrag),
+ srcvalue, (vti.Mask true_mask), VLOpFrag)),
(!cast<Instruction>(instruction_name#"_VX_" # vti.LMul.MX)
wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
@@ -903,12 +926,12 @@ multiclass VPatNarrowShiftSplat_WX_WI<SDNode op, string instruction_name> {
defvar wti = vtiTowti.Wti;
def : Pat<(vti.Vector (riscv_trunc_vector_vl
(wti.Vector (op wti.RegClass:$rs1, (SplatPat XLenVT:$rs2),
- true_mask, VLOpFrag)), true_mask, VLOpFrag)),
+ srcvalue, true_mask, VLOpFrag)), true_mask, VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WX_"#vti.LMul.MX)
wti.RegClass:$rs1, GPR:$rs2, GPR:$vl, vti.Log2SEW)>;
def : Pat<(vti.Vector (riscv_trunc_vector_vl
(wti.Vector (op wti.RegClass:$rs1, (SplatPat_uimm5 uimm5:$rs2),
- true_mask, VLOpFrag)), true_mask, VLOpFrag)),
+ srcvalue, true_mask, VLOpFrag)), true_mask, VLOpFrag)),
(!cast<Instruction>(instruction_name#"_WI_"#vti.LMul.MX)
wti.RegClass:$rs1, uimm5:$rs2, GPR:$vl, vti.Log2SEW)>;
}
@@ -991,16 +1014,16 @@ defm : VPatBinaryVL_VV_VX<riscv_sub_vl, "PseudoVSUB">;
// pattern operands
foreach vti = AllIntegerVectors in {
def : Pat<(riscv_sub_vl (vti.Vector (SplatPat (XLenVT GPR:$rs2))),
- (vti.Vector vti.RegClass:$rs1), (vti.Mask V0),
- VLOpFrag),
+ (vti.Vector vti.RegClass:$rs1),
+ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVRSUB_VX_"# vti.LMul.MX#"_MASK")
- (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, GPR:$rs2,
+ vti.RegClass:$merge, vti.RegClass:$rs1, GPR:$rs2,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(riscv_sub_vl (vti.Vector (SplatPat_simm5 simm5:$rs2)),
- (vti.Vector vti.RegClass:$rs1), (vti.Mask V0),
- VLOpFrag),
+ (vti.Vector vti.RegClass:$rs1),
+ vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVRSUB_VI_"# vti.LMul.MX#"_MASK")
- (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, simm5:$rs2,
+ vti.RegClass:$merge, vti.RegClass:$rs1, simm5:$rs2,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
}
@@ -1038,8 +1061,7 @@ foreach vti = AllIntegerVectors in {
// Emit shift by 1 as an add since it might be faster.
def : Pat<(riscv_shl_vl (vti.Vector vti.RegClass:$rs1),
(riscv_vmv_v_x_vl (vti.Vector undef), 1, (XLenVT srcvalue)),
- (vti.Mask true_mask),
- VLOpFrag),
+ srcvalue, (vti.Mask true_mask), VLOpFrag),
(!cast<Instruction>("PseudoVADD_VV_"# vti.LMul.MX)
vti.RegClass:$rs1, vti.RegClass:$rs1, GPR:$vl, vti.Log2SEW)>;
}
@@ -1140,8 +1162,10 @@ foreach vtiTowti = AllWidenableIntVectors in {
(riscv_add_vl wti.RegClass:$rd,
(riscv_vwmulsu_vl_oneuse (vti.Vector vti.RegClass:$rs1),
(SplatPat XLenVT:$rs2),
- (vti.Mask true_mask), VLOpFrag),
- (vti.Mask true_mask), VLOpFrag)),
+ srcvalue,
+ (vti.Mask true_mask),
+ VLOpFrag),
+ srcvalue, (vti.Mask true_mask),VLOpFrag)),
(!cast<Instruction>("PseudoVWMACCUS_VX_" # vti.LMul.MX)
wti.RegClass:$rd, vti.ScalarRegClass:$rs2, vti.RegClass:$rs1,
GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
More information about the llvm-commits
mailing list