[llvm] [SDAG] Add basic type legalization for experimental.vector.match (PR #213344)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 05:17:15 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/213344
>From 078d14ab98c363d854f7e6460fe2ad0d2f631a40 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 31 Jul 2026 12:11:35 +0000
Subject: [PATCH 1/3] [SDAG] Add basic type legalization for
experimental.vector.match
This adds a new ISD::VECTOR_MATCH opcode and implements basic type
legalization. Currently, this operation is only used on AArch64 (as the
only target with any form of match instruction).
Right now, this does not support many more cases than the existing
lowering. A follow up patch will handle supporting needle sizes <= the
maximum size without falling back to expansion.
Assisted-by: Codex
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 8 +-
llvm/include/llvm/CodeGen/ISDOpcodes.h | 7 +
llvm/include/llvm/CodeGen/TargetLowering.h | 14 +-
.../include/llvm/Target/TargetSelectionDAG.td | 7 +
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 22 ++
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 8 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 137 +++++++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 3 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 31 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 83 +++---
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 3 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 36 ++-
.../AArch64/intrinsic-vector-match-sve2.ll | 270 ++++--------------
15 files changed, 364 insertions(+), 292 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..6c7bb56563ab6 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2572,14 +2572,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
unsigned SearchSize = NeedleTy->getNumElements();
- // If we're not expanding the intrinsic then we assume this is cheap to
- // implement.
- EVT SearchVT = getTLI()->getValueType(DL, SearchTy);
- if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize))
- return getTypeLegalizationCost(RetTy).first;
-
// Approximate the cost based on the expansion code in
- // SelectionDAGBuilder.
+ // TargetLowering::expandVectorMatch.
InstructionCost Cost = 0;
Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, NeedleTy,
CostKind, 1, nullptr, nullptr);
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 23ef5d22963ae..7124553a1b4a8 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1627,6 +1627,13 @@ enum NodeType {
/// bits conform to getBooleanContents similar to the SETCC operator.
GET_ACTIVE_LANE_MASK,
+ /// VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match
+ /// intrinsic.
+ /// Operands: Source, Needle, Mask
+ /// Source has the same number of elements as the result and Needle may have
+ /// a different number of elements. The result type matches Mask.
+ VECTOR_MATCH,
+
/// The `llvm.loop.dependence.{war, raw}.mask` intrinsics
/// Operands: Load pointer, Store pointer, Element size, Lane offset
/// Output: Mask
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c663bb8ea65b7..44a43581b8b25 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -505,6 +505,8 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
+ virtual bool shouldExpandVectorMatch() const { return true; }
+
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
@@ -520,13 +522,6 @@ class LLVM_ABI TargetLoweringBase {
bool ZeroIsPoison,
const ConstantRange *VScaleRange) const;
- /// Return true if the @llvm.experimental.vector.match intrinsic should be
- /// expanded for vector type `VT' and search size `SearchSize' using generic
- /// code in SelectionDAGBuilder.
- virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const {
- return true;
- }
-
// Return true if op(vecreduce(x), vecreduce(y)) should be reassociated to
// vecreduce(op(x, y)) for the reduction opcode RedOpc.
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const {
@@ -5765,6 +5760,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// \returns The expansion result or SDValue() if it fails.
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand VECTOR_MATCH nodes.
+ /// \param N Node to expand
+ /// \returns The expansion result or SDValue() if it fails.
+ SDValue expandVectorMatch(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand VECTOR_FIND_LAST_ACTIVE nodes
/// \param N Node to expand
/// \returns The expansion result or SDValue() if it fails.
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index c0b3b3008bcf7..4b367c00535d6 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -931,6 +931,13 @@ def get_active_lane_mask
SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<1, 2>]>,
[]>;
+def vector_match
+ : SDNode<"ISD::VECTOR_MATCH",
+ SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>,
+ SDTCisVec<3>, SDTCisSameNumEltsAs<0, 1>,
+ SDTCisSameAs<0, 3>]>,
+ []>;
+
// Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
// these internally. Don't reference these directly.
def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 1903ea89e4af8..b63ee2845c5b9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4635,6 +4635,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VP_CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandVPCTTZElements(Node, DAG));
break;
+ case ISD::VECTOR_MATCH:
+ Results.push_back(TLI.expandVectorMatch(Node, DAG));
+ break;
case ISD::CLEAR_CACHE:
// The default expansion of llvm.clear_cache is simply a no-op for those
// targets where it is not needed.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 2fee0b280ed9b..59b2098ea9bac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -168,6 +168,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = PromoteIntRes_VECTOR_MATCH(N);
+ break;
case ISD::PARTIAL_REDUCE_UMLA:
case ISD::PARTIAL_REDUCE_SMLA:
@@ -2293,6 +2296,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = PromoteIntOp_VECTOR_MATCH(N, OpNo);
+ break;
case ISD::MASKED_UDIV:
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
@@ -3166,6 +3172,16 @@ SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ assert(OpNo < 3 && "Unexpected operand for promotion");
+ if (OpNo != 2)
+ return TLI.expandVectorMatch(N, DAG);
+
+ SmallVector<SDValue, 3> NewOps(N->ops());
+ NewOps[2] = PromoteTargetBoolean(N->getOperand(2), N->getValueType(0));
+ return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
assert(OpNo == 2);
SmallVector<SDValue, 3> NewOps(N->ops());
@@ -6605,6 +6621,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_MATCH(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getNode(ISD::VECTOR_MATCH, SDLoc(N), NVT, N->ops());
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
SDLoc DL(N);
EVT VT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index d4d56a9563f71..cfd23912871c3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -367,6 +367,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_READ_REGISTER(SDNode *N);
SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_MATCH(SDNode *N);
SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N);
@@ -425,6 +426,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_UnaryBooleanVectorOp(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N);
SDValue PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo);
@@ -895,6 +897,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecOp_FAKE_USE(SDNode *N);
SDValue ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
SDValue ScalarizeVecOp_CTTZ_ELTS(SDNode *N);
+ SDValue ScalarizeVecRes_VECTOR_MATCH(SDNode *N);
+ SDValue ScalarizeVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
SDValue ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
@@ -975,6 +979,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo, SDValue &Hi);
// Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
bool SplitVectorOperand(SDNode *N, unsigned OpNo);
@@ -1010,6 +1015,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_VECTOR_HISTOGRAM(SDNode *N);
SDValue SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
// Vector Widening Support: LegalizeVectorTypes.cpp
@@ -1078,6 +1084,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);
SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue WidenVecRes_VECTOR_MATCH(SDNode *N);
void WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N);
SDValue WidenVecRes_Ternary(SDNode *N);
@@ -1135,6 +1142,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecOp_CttzElements(SDNode *N);
SDValue WidenVecOp_VP_CttzElements(SDNode *N);
SDValue WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
/// Helper function to generate a set of operations to perform
/// a vector operation for a wider type.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index c71352fb20817..4576e054d7337 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -89,6 +89,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
+ case ISD::VECTOR_MATCH:
+ R = ScalarizeVecRes_VECTOR_MATCH(N);
+ break;
case ISD::POISON:
case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
@@ -965,6 +968,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::CTTZ_ELTS_ZERO_POISON:
Res = ScalarizeVecOp_CTTZ_ELTS(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
+ break;
case ISD::MASKED_UDIV:
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
@@ -1304,6 +1310,20 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
return DAG.getZExtOrTrunc(SetCC, SDLoc(N), N->getValueType(0));
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
+ SDLoc DL(N);
+ // Reuse the expansion (which should scalarize).
+ SDValue Mask = TLI.expandVectorMatch(N, DAG);
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
+ N->getValueType(0).getScalarType(), Mask,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
+ unsigned OpNo) {
+ return TLI.expandVectorMatch(N, DAG);
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
assert(OpNo == 2 && "Can only scalarize mask operand");
SDLoc DL(N);
@@ -1652,6 +1672,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
break;
+ case ISD::VECTOR_MATCH:
+ SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
+ break;
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -3732,6 +3755,20 @@ void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
Hi = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, HiVT, HiStartVal, Op1);
}
+void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ SDValue SourceLo, SourceHi;
+ GetSplitVector(N->getOperand(0), SourceLo, SourceHi);
+ SDValue MaskLo, MaskHi;
+ GetSplitVector(N->getOperand(2), MaskLo, MaskHi);
+ SDLoc DL(N);
+
+ Lo = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
+ N->getOperand(1), MaskLo, N->getFlags());
+ Hi = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
+ N->getOperand(1), MaskHi, N->getFlags());
+}
+
void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
unsigned Factor = N->getNumOperands();
@@ -3970,6 +4007,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::PARTIAL_REDUCE_FMLA:
Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
+ break;
}
// If the result is null, the sub-method took care of registering results etc.
@@ -5164,6 +5204,44 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
MMO, IndexType);
}
+SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ SDLoc DL(N);
+
+ if (OpNo != 1) {
+ auto SplitOperand = [&](SDValue Op, SDValue &Lo, SDValue &Hi) {
+ if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSplitVector)
+ GetSplitVector(Op, Lo, Hi);
+ else
+ std::tie(Lo, Hi) = DAG.SplitVector(Op, DL);
+ };
+
+ SDValue SourceLo, SourceHi;
+ SplitOperand(N->getOperand(0), SourceLo, SourceHi);
+ SDValue MaskLo, MaskHi;
+ SplitOperand(N->getOperand(2), MaskLo, MaskHi);
+
+ SDValue MatchLo =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
+ N->getOperand(1), MaskLo, N->getFlags());
+ SDValue MatchHi =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
+ N->getOperand(1), MaskHi, N->getFlags());
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), MatchLo,
+ MatchHi);
+ }
+
+ SDValue NeedleLo, NeedleHi;
+ GetSplitVector(N->getOperand(1), NeedleLo, NeedleHi);
+
+ SDValue MatchLo =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
+ NeedleLo, N->getOperand(2), N->getFlags());
+ SDValue MatchHi =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
+ NeedleHi, N->getOperand(2), N->getFlags());
+ return DAG.getNode(ISD::OR, DL, N->getValueType(0), MatchLo, MatchHi);
+}
+
SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
SDValue Acc = N->getOperand(0);
assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
@@ -5306,6 +5384,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = WidenVecRes_VECTOR_MATCH(N);
+ break;
case ISD::VECTOR_DEINTERLEAVE:
WidenVecRes_VECTOR_DEINTERLEAVE(N);
break;
@@ -7463,6 +7544,22 @@ SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
}
+SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
+ SDLoc DL(N);
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+ EVT SourceVT = N->getOperand(0).getValueType();
+ EVT WideSourceVT =
+ EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
+ WidenVT.getVectorElementCount());
+
+ SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
+ N->getOperand(0), 0);
+ SDValue WideMask = DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, WidenVT),
+ N->getOperand(2), 0);
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
+ N->getOperand(1), WideMask, N->getFlags());
+}
+
void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
EVT VT = N->getValueType(0);
EVT EltVT = VT.getVectorElementType();
@@ -7735,6 +7832,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_FIND_LAST_ACTIVE:
Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
+ break;
}
// If Res is null, the sub-method took care of registering the result.
@@ -8761,6 +8861,43 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
WideMask);
}
+SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ if (OpNo != 1) {
+ SDLoc DL(N);
+ EVT ResVT = N->getValueType(0);
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
+ EVT SourceVT = N->getOperand(0).getValueType();
+ EVT WideSourceVT =
+ EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
+ WidenVT.getVectorElementCount());
+
+ SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
+ N->getOperand(0), 0);
+ SDValue WideMask = DAG.getInsertSubvector(
+ DL, DAG.getConstant(0, DL, WidenVT), N->getOperand(2), 0);
+ SDValue WideMatch = DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
+ N->getOperand(1), WideMask, N->getFlags());
+ return DAG.getExtractSubvector(DL, ResVT, WideMatch, 0);
+ }
+
+ SDLoc DL(N);
+ SDValue Needle = N->getOperand(1);
+ EVT NeedleVT = Needle.getValueType();
+ if (NeedleVT.getVectorNumElements() == 1)
+ return TLI.expandVectorMatch(N, DAG);
+
+ EVT WidenNeedleVT = TLI.getTypeToTransformTo(*DAG.getContext(), NeedleVT);
+
+ SDValue Fill =
+ DAG.getExtractVectorElt(DL, NeedleVT.getVectorElementType(), Needle, 0);
+ SDValue WideNeedle = DAG.getSplatVector(WidenNeedleVT, DL, Fill);
+ WideNeedle = DAG.getInsertSubvector(DL, WideNeedle, Needle, 0);
+
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0),
+ N->getOperand(0), WideNeedle, N->getOperand(2),
+ N->getFlags());
+}
+
//===----------------------------------------------------------------------===//
// Vector Widening Utilities
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f1c9809c2159e..0f4011f9fc6ab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8526,30 +8526,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));
SDValue Mask = getValue(I.getOperand(2));
- EVT Op1VT = Op1.getValueType();
- EVT Op2VT = Op2.getValueType();
EVT ResVT = Mask.getValueType();
- unsigned SearchSize = Op2VT.getVectorNumElements();
-
- // If the target has native support for this vector match operation, lower
- // the intrinsic untouched; otherwise, expand it below.
- if (!TLI.shouldExpandVectorMatch(Op1VT, SearchSize)) {
- visitTargetIntrinsic(I, Intrinsic);
+ if (!TLI.shouldExpandVectorMatch()) {
+ setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
return;
}
- SDValue Ret = DAG.getConstant(0, sdl, ResVT);
-
- for (unsigned i = 0; i < SearchSize; ++i) {
- SDValue Op2Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl,
- Op2VT.getVectorElementType(), Op2,
- DAG.getVectorIdxConstant(i, sdl));
- SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, sdl, Op1VT, Op2Elem);
- SDValue Cmp = DAG.getSetCC(sdl, ResVT, Op1, Splat, ISD::SETEQ);
- Ret = DAG.getNode(ISD::OR, sdl, ResVT, Ret, Cmp);
- }
-
- setValue(&I, DAG.getNode(ISD::AND, sdl, ResVT, Ret, Mask));
+ SDValue Match = DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask);
+ setValue(&I, TLI.expandVectorMatch(Match.getNode(), DAG));
return;
}
case Intrinsic::vector_reverse:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 028f0b9486c01..5c1b280697262 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -610,6 +610,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::GET_ACTIVE_LANE_MASK:
return "get_active_lane_mask";
+ case ISD::VECTOR_MATCH:
+ return "vector_match";
+
case ISD::PARTIAL_REDUCE_UMLA:
return "partial_reduce_umla";
case ISD::PARTIAL_REDUCE_SMLA:
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index b82f42493f57d..53395a1b1b690 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13836,6 +13836,37 @@ SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
return DAG.getZExtOrTrunc(Sub, DL, VT);
}
+SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
+ SDLoc DL(N);
+ SDValue Source = N->getOperand(0);
+ SDValue Needle = N->getOperand(1);
+ SDValue Mask = N->getOperand(2);
+ EVT SourceVT = Source.getValueType();
+ EVT NeedleVT = Needle.getValueType();
+ EVT ResVT = N->getValueType(0);
+ EVT CmpVT =
+ getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SourceVT);
+
+ assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
+
+ SDValue Ret = DAG.getConstant(0, DL, CmpVT);
+ EVT NeedleEltVT = NeedleVT.getVectorElementType();
+ for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
+ SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
+ SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
+ Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
+ }
+
+ if (Mask.getValueType() != CmpVT)
+ Mask = DAG.getBoolExtOrTrunc(Mask, DL, CmpVT, Mask.getValueType());
+
+ Ret = DAG.getNode(ISD::AND, DL, CmpVT, Ret, Mask);
+ if (Ret.getValueType() != ResVT)
+ Ret = DAG.getBoolExtOrTrunc(Ret, DL, ResVT, Ret.getValueType());
+ return Ret;
+}
+
SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
SelectionDAG &DAG) const {
SDLoc DL(N);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a5a2c9e430ac2..ebac1022c8fb9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1646,6 +1646,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
}
+ for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+
// FIXME: Move lowering for more nodes here if those are common between
// SVE and SME.
if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -1661,6 +1664,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+ }
+ if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
+ for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
+
+ for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
}
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
@@ -2487,6 +2498,8 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
return false;
}
+bool AArch64TargetLowering::shouldExpandVectorMatch() const { return false; }
+
bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
if (!Subtarget->isSVEorStreamingSVEAvailable())
return true;
@@ -2498,19 +2511,6 @@ bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
VT != MVT::v4i1 && VT != MVT::v2i1;
}
-bool AArch64TargetLowering::shouldExpandVectorMatch(EVT VT,
- unsigned SearchSize) const {
- // MATCH is SVE2 and only available in non-streaming mode.
- if (!Subtarget->hasSVE2() || !Subtarget->isSVEAvailable())
- return true;
- // Furthermore, we can only use it for 8-bit or 16-bit elements.
- if (VT == MVT::nxv8i16 || VT == MVT::v8i16)
- return SearchSize != 8;
- if (VT == MVT::nxv16i8 || VT == MVT::v16i8 || VT == MVT::v8i8)
- return SearchSize != 8 && SearchSize != 16;
- return true;
-}
-
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
@@ -6496,23 +6496,32 @@ SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
DAG.getTargetConstant(ImmAddend, DL, MVT::i32)});
}
-SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
+SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG,
+ const AArch64Subtarget *Subtarget) {
SDLoc DL(Op);
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
-
- auto Op1 = Op.getOperand(1);
- auto Op2 = Op.getOperand(2);
- auto Mask = Op.getOperand(3);
+ auto Op1 = Op.getOperand(0);
+ auto Op2 = Op.getOperand(1);
+ auto Mask = Op.getOperand(2);
EVT Op1VT = Op1.getValueType();
EVT Op2VT = Op2.getValueType();
EVT ResVT = Op.getValueType();
+ unsigned SearchSize = Op2VT.getVectorNumElements();
assert((Op1VT.getVectorElementType() == MVT::i8 ||
Op1VT.getVectorElementType() == MVT::i16) &&
"Expected 8-bit or 16-bit characters.");
+ if ((Op1VT == MVT::nxv8i16 || Op1VT == MVT::v8i16) && SearchSize != 8)
+ return SDValue();
+
+ if ((Op1VT == MVT::nxv16i8 || Op1VT == MVT::v16i8 || Op1VT == MVT::v8i8) &&
+ SearchSize != 8 && SearchSize != 16)
+ return SDValue();
+
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
+
// Scalable vector type used to wrap operands.
// A single container is enough for both operands because ultimately the
// operands will have to be wrapped to the same type (nxv16i8 or nxv8i16).
@@ -7236,9 +7245,6 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
ADDLV, DAG.getConstant(0, DL, MVT::i64));
return EXTRACT_VEC_ELT;
}
- case Intrinsic::experimental_vector_match: {
- return LowerVectorMatch(Op, DAG);
- }
case Intrinsic::aarch64_cls:
case Intrinsic::aarch64_cls64: {
SDValue Res = DAG.getNode(ISD::CTLS, DL, Op.getOperand(1).getValueType(),
@@ -8883,6 +8889,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_INTERLEAVE(Op, DAG);
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
+ case ISD::VECTOR_MATCH:
+ return LowerVectorMatch(Op, DAG, Subtarget);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
@@ -32304,6 +32312,21 @@ void AArch64TargetLowering::ReplaceNodeResults(
case ISD::GET_ACTIVE_LANE_MASK:
ReplaceGetActiveLaneMaskResults(N, Results, DAG);
return;
+ case ISD::VECTOR_MATCH: {
+ EVT VT = N->getValueType(0);
+ if (!VT.isFixedLengthVectorOf(MVT::i1))
+ return;
+
+ // NOTE: Only trivial type promotion is supported.
+ EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
+ if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
+ return;
+
+ SDLoc DL(N);
+ SDValue V = DAG.getNode(ISD::VECTOR_MATCH, DL, NewVT, N->ops());
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
+ return;
+ }
case ISD::INTRINSIC_WO_CHAIN: {
EVT VT = N->getValueType(0);
@@ -32360,20 +32383,6 @@ void AArch64TargetLowering::ReplaceNodeResults(
DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, RuntimePStateSM));
return;
}
- case Intrinsic::experimental_vector_match: {
- if (!VT.isFixedLengthVectorOf(MVT::i1))
- return;
-
- // NOTE: Only trivial type promotion is supported.
- EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
- if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
- return;
-
- SDLoc DL(N);
- auto V = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, NewVT, N->ops());
- Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
- return;
- }
}
}
case ISD::READ_REGISTER: {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 35f7ef0e2151e..5a07d7e2c9e3e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -565,11 +565,10 @@ class AArch64TargetLowering : public TargetLowering {
bool AllowUnknown = false) const override;
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
+ bool shouldExpandVectorMatch() const override;
bool shouldExpandCttzElements(EVT VT) const override;
- bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const override;
-
/// If a change in streaming mode is required on entry to/return from a
/// function call it emits and returns the corresponding SMSTART or SMSTOP
/// node. \p Condition should be one of the enum values from
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 03aa42b31e6aa..dc6140c219805 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1149,18 +1149,30 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
EVT SearchVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
unsigned SearchSize = NeedleTy->getNumElements();
- if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize)) {
- // Base cost for MATCH instructions. At least on the Neoverse V2 and
- // Neoverse V3, these are cheap operations with the same latency as a
- // vector ADD. In most cases, however, we also need to do an extra DUP.
- // For fixed-length vectors we currently need an extra five--six
- // instructions besides the MATCH.
- InstructionCost Cost = 4;
- if (isa<FixedVectorType>(RetTy))
- Cost += 10;
- return Cost;
- }
- break;
+ auto IsSupportedTypeAndSearchSize = [&]() {
+ if (SearchVT == MVT::nxv8i16 || SearchVT == MVT::v8i16)
+ return SearchSize == 8;
+
+ if (SearchVT == MVT::nxv16i8 || SearchVT == MVT::v16i8 ||
+ SearchVT == MVT::v8i8)
+ return SearchSize == 8 || SearchSize == 16;
+
+ return false;
+ };
+
+ if (!ST->hasSVE2() || !ST->isSVEAvailable() ||
+ !IsSupportedTypeAndSearchSize())
+ break;
+
+ // Base cost for MATCH instructions. At least on the Neoverse V2 and
+ // Neoverse V3, these are cheap operations with the same latency as a
+ // vector ADD. In most cases, however, we also need to do an extra DUP.
+ // For fixed-length vectors we currently need an extra five--six
+ // instructions besides the MATCH.
+ InstructionCost Cost = 4;
+ if (isa<FixedVectorType>(RetTy))
+ Cost += 10;
+ return Cost;
}
case Intrinsic::cttz: {
auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index de88e10141fbf..1f4101c6b1b56 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -12,6 +12,22 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
ret <vscale x 16 x i1> %r
}
+define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #0 {
+; CHECK-LABEL: match_v1i8_v1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: umov w8, v1.b[0]
+; CHECK-NEXT: dup v1.8b, w8
+; CHECK-NEXT: sbfx w8, w0, #0, #1
+; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: umov w0, v0.b[0]
+; CHECK-NEXT: ret
+ %r = tail call <1 x i1> @llvm.experimental.vector.match(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask)
+ ret <1 x i1> %r
+}
+
define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v2i8:
; CHECK: // %bb.0:
@@ -88,9 +104,11 @@ define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mas
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -104,8 +122,10 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: shl v1.16b, v2.16b, #7
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -119,6 +139,7 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v4.16b, v1.b[0]
; CHECK-NEXT: dup v5.16b, v1.b[4]
; CHECK-NEXT: dup v1.16b, v1.b[6]
+; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
@@ -126,7 +147,8 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -231,112 +253,13 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v32i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
-; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov z3.b, z1.b[1]
-; CHECK-NEXT: mov z4.b, b1
; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: mov z5.b, z1.b[2]
-; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z3.b, z1.b[3]
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z5.b
-; CHECK-NEXT: mov z4.b, z1.b[4]
-; CHECK-NEXT: mov p2.b, p3/m, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[5]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[6]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[7]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[8]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[9]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[10]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[11]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[12]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[13]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[14]
-; CHECK-NEXT: mov z1.b, z1.b[15]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, b2
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[1]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[2]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[3]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[4]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[5]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[6]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[7]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[8]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[9]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[10]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[11]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[12]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[13]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[14]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[15]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: cmpeq p1.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
-; CHECK-NEXT: orr p0.b, p0/z, p2.b, p1.b
-; CHECK-NEXT: addvl sp, sp, #1
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: mov z1.q, q1
+; CHECK-NEXT: match p1.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: match p2.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: sel p0.b, p2, p2.b, p1.b
; CHECK-NEXT: ret
%r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <32 x i8> %op2, <vscale x 16 x i1> %mask)
ret <vscale x 16 x i1> %r
@@ -345,107 +268,34 @@ define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8
define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask) #0 {
; CHECK-LABEL: match_v16i8_v32i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: dup v4.16b, v1.b[1]
-; CHECK-NEXT: dup v5.16b, v1.b[0]
-; CHECK-NEXT: dup v6.16b, v1.b[2]
-; CHECK-NEXT: dup v7.16b, v1.b[3]
-; CHECK-NEXT: dup v16.16b, v1.b[4]
-; CHECK-NEXT: dup v17.16b, v1.b[5]
-; CHECK-NEXT: dup v18.16b, v1.b[6]
-; CHECK-NEXT: dup v19.16b, v1.b[7]
-; CHECK-NEXT: dup v20.16b, v1.b[8]
-; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
-; CHECK-NEXT: cmeq v6.16b, v0.16b, v6.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v7.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v16.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v17.16b
-; CHECK-NEXT: dup v21.16b, v2.b[7]
-; CHECK-NEXT: dup v22.16b, v1.b[10]
-; CHECK-NEXT: orr v4.16b, v5.16b, v4.16b
-; CHECK-NEXT: orr v5.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v18.16b, v1.b[9]
-; CHECK-NEXT: dup v19.16b, v1.b[11]
-; CHECK-NEXT: dup v20.16b, v1.b[12]
-; CHECK-NEXT: cmeq v22.16b, v0.16b, v22.16b
-; CHECK-NEXT: orr v4.16b, v4.16b, v5.16b
-; CHECK-NEXT: orr v5.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: dup v18.16b, v1.b[13]
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v19.16b, v2.b[0]
-; CHECK-NEXT: dup v20.16b, v2.b[1]
-; CHECK-NEXT: orr v4.16b, v4.16b, v5.16b
-; CHECK-NEXT: dup v5.16b, v2.b[6]
-; CHECK-NEXT: orr v6.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v7.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v19.16b, v2.b[2]
-; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[8]
-; CHECK-NEXT: orr v6.16b, v6.16b, v22.16b
-; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT: dup v16.16b, v1.b[14]
-; CHECK-NEXT: dup v1.16b, v1.b[15]
-; CHECK-NEXT: orr v17.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v19.16b, v2.b[3]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[9]
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v16.16b
-; CHECK-NEXT: cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT: orr v4.16b, v4.16b, v6.16b
-; CHECK-NEXT: orr v17.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v19.16b, v2.b[4]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[10]
-; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT: orr v16.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v18.16b, v2.b[5]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v19.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v20.16b, v2.b[11]
-; CHECK-NEXT: orr v1.16b, v7.16b, v1.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: dup v17.16b, v2.b[12]
-; CHECK-NEXT: orr v5.16b, v5.16b, v19.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v18.16b, v2.b[13]
-; CHECK-NEXT: dup v19.16b, v2.b[14]
-; CHECK-NEXT: orr v1.16b, v4.16b, v1.16b
-; CHECK-NEXT: dup v2.16b, v2.b[15]
-; CHECK-NEXT: orr v4.16b, v6.16b, v7.16b
-; CHECK-NEXT: cmeq v6.16b, v0.16b, v17.16b
-; CHECK-NEXT: orr v5.16b, v5.16b, v16.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v4.16b, v5.16b, v6.16b
-; CHECK-NEXT: orr v5.16b, v7.16b, v16.16b
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
+; CHECK-NEXT: shl v3.16b, v3.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmpne p1.b, p0/z, z3.b, #0
+; CHECK-NEXT: match p0.b, p1/z, z0.b, z2.b
+; CHECK-NEXT: match p2.b, p1/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v3.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
}
+define <vscale x 32 x i1> @match_nxv32i8_v16i8(<vscale x 32 x i8> %op1, <16 x i8> %op2, <vscale x 32 x i1> %mask) #0 {
+; CHECK-LABEL: match_nxv32i8_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: match p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: match p1.b, p1/z, z1.b, z2.b
+; CHECK-NEXT: ret
+ %r = tail call <vscale x 32 x i1> @llvm.experimental.vector.match(<vscale x 32 x i8> %op1, <16 x i8> %op2, <vscale x 32 x i1> %mask)
+ ret <vscale x 32 x i1> %r
+}
+
; Data types not supported by MATCH.
; Note: The cases for SVE could be made tighter.
@@ -504,11 +354,14 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s
; CHECK-NEXT: cmeq v5.4s, v0.4s, v5.4s
; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
+; CHECK-NEXT: shl v1.4h, v2.4h, #15
+; CHECK-NEXT: orr v2.16b, v4.16b, v3.16b
+; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
-; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
; CHECK-NEXT: ret
%r = tail call <4 x i1> @llvm.experimental.vector.match(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
ret <4 x i1> %r
@@ -517,13 +370,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) #0 {
; CHECK-LABEL: match_v2xi64_v2i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: shl v2.2s, v2.2s, #31
; CHECK-NEXT: dup v3.2d, v1.d[1]
; CHECK-NEXT: dup v1.2d, v1.d[0]
+; CHECK-NEXT: cmlt v2.2s, v2.2s, #0
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: sshll v1.2d, v2.2s, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
>From 28824c6f6068bf9f9e462d462fb2d4f464d4320a Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 3 Aug 2026 08:58:00 +0000
Subject: [PATCH 2/3] Fixups
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 -
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 -
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 -
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 -
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +
.../RISCV/rvv/intrinsic-vector-match.ll | 555 ++++++++++++------
8 files changed, 367 insertions(+), 211 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 44a43581b8b25..1486552624d8b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -505,8 +505,6 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
- virtual bool shouldExpandVectorMatch() const { return true; }
-
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index b63ee2845c5b9..1903ea89e4af8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4635,9 +4635,6 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VP_CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandVPCTTZElements(Node, DAG));
break;
- case ISD::VECTOR_MATCH:
- Results.push_back(TLI.expandVectorMatch(Node, DAG));
- break;
case ISD::CLEAR_CACHE:
// The default expansion of llvm.clear_cache is simply a no-op for those
// targets where it is not needed.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7b97c41ac2dbc..44fc8c0d41172 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -489,6 +489,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
case ISD::MASKED_SREM:
+ case ISD::VECTOR_MATCH:
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
case ISD::SMULFIX:
@@ -1324,6 +1325,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
+ case ISD::VECTOR_MATCH:
+ Results.push_back(TLI.expandVectorMatch(Node, DAG));
+ return;
case ISD::SREM:
case ISD::UREM:
ExpandREM(Node, Results);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0f4011f9fc6ab..fddb5f0c247d9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8527,13 +8527,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Op2 = getValue(I.getOperand(1));
SDValue Mask = getValue(I.getOperand(2));
EVT ResVT = Mask.getValueType();
- if (!TLI.shouldExpandVectorMatch()) {
- setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
- return;
- }
-
- SDValue Match = DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask);
- setValue(&I, TLI.expandVectorMatch(Match.getNode(), DAG));
+ setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
return;
}
case Intrinsic::vector_reverse:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ebac1022c8fb9..aebd304fccd2b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2498,8 +2498,6 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
return false;
}
-bool AArch64TargetLowering::shouldExpandVectorMatch() const { return false; }
-
bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
if (!Subtarget->isSVEorStreamingSVEAvailable())
return true;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 5a07d7e2c9e3e..1b5587f271df9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -565,8 +565,6 @@ class AArch64TargetLowering : public TargetLowering {
bool AllowUnknown = false) const override;
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
- bool shouldExpandVectorMatch() const override;
-
bool shouldExpandCttzElements(EVT VT) const override;
/// If a change in streaming mode is required on entry to/return from a
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 37505a5b187f8..aa5dd5f7cff4b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1038,6 +1038,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (!isTypeLegal(VT))
continue;
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
// Mask VTs are custom-expanded into a series of standard nodes
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index 6febcd22e3288..d341093bd131f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -225,10 +225,26 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-LABEL: match_v16i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 8
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 9
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 10
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 11
+; CHECK-NEXT: vmv.x.s a3, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vmv.x.s a4, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vmv.x.s a5, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vmv.x.s a7, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
@@ -245,32 +261,24 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-NEXT: vmseq.vv v12, v8, v13
; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vrgather.vi v11, v9, 7
-; CHECK-NEXT: vrgather.vi v13, v9, 8
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 9
-; CHECK-NEXT: vrgather.vi v13, v9, 10
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 11
-; CHECK-NEXT: vrgather.vi v13, v9, 12
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 13
-; CHECK-NEXT: vrgather.vi v13, v9, 14
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 15
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a3
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a4
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a5
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a6
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a7
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -282,28 +290,32 @@ define <8 x i1> @match_v8i8_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i8_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
+; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vrgather.vi v12, v9, 2
; CHECK-NEXT: vmor.mm v10, v11, v10
-; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 5
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 7
+; CHECK-NEXT: vrgather.vi v12, v9, 3
; CHECK-NEXT: vmor.mm v9, v10, v11
-; CHECK-NEXT: vmseq.vv v8, v8, v12
+; CHECK-NEXT: vmseq.vv v10, v8, v12
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -348,28 +360,32 @@ define <8 x i1> @match_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
+; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vrgather.vi v13, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 5
-; CHECK-NEXT: vrgather.vi v13, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -383,22 +399,30 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-LABEL: match_v8i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 8
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 9
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 11
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
; CHECK-NEXT: vmv.x.s a3, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vslidedown.vi v10, v9, 8
; CHECK-NEXT: vmv.x.s a4, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vslidedown.vi v10, v9, 9
; CHECK-NEXT: vmv.x.s a5, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vslidedown.vi v10, v9, 10
; CHECK-NEXT: vmv.x.s a6, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vslidedown.vi v10, v9, 11
; CHECK-NEXT: vmv.x.s a7, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vmv.x.s t0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vmv.x.s t1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vmv.x.s t2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vmv.x.s t3, v10
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
@@ -408,18 +432,6 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vmseq.vv v11, v8, v12
; CHECK-NEXT: vrgather.vi v12, v9, 3
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 5
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v11
; CHECK-NEXT: vmseq.vv v10, v8, v12
; CHECK-NEXT: vmseq.vx v11, v8, a0
@@ -436,8 +448,16 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v9, v9, v11
; CHECK-NEXT: vmseq.vx v11, v8, a6
; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a7
; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a7
+; CHECK-NEXT: vmseq.vx v11, v8, t0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, t1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, t2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, t3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -712,43 +732,76 @@ define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8
define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask) {
; RV32-LABEL: match_v16i8_v32i8:
; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: addi sp, sp, -48
+; RV32-NEXT: .cfi_def_cfa_offset 48
+; RV32-NEXT: sw s0, 44(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s1, 40(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s3, 32(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s5, 24(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s6, 20(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s7, 16(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s8, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset s0, -4
+; RV32-NEXT: .cfi_offset s1, -8
+; RV32-NEXT: .cfi_offset s2, -12
+; RV32-NEXT: .cfi_offset s3, -16
+; RV32-NEXT: .cfi_offset s4, -20
+; RV32-NEXT: .cfi_offset s5, -24
+; RV32-NEXT: .cfi_offset s6, -28
+; RV32-NEXT: .cfi_offset s7, -32
+; RV32-NEXT: .cfi_offset s8, -36
+; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v9, v10, 8
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 9
+; RV32-NEXT: vmv.x.s a1, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 10
+; RV32-NEXT: vmv.x.s a2, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 11
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 12
+; RV32-NEXT: vmv.x.s a4, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 13
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 14
+; RV32-NEXT: vmv.x.s a6, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 15
+; RV32-NEXT: vmv.x.s a7, v9
; RV32-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV32-NEXT: vslidedown.vi v12, v10, 16
-; RV32-NEXT: vmv.x.s a0, v12
+; RV32-NEXT: vmv.x.s t0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 17
-; RV32-NEXT: vmv.x.s a1, v12
+; RV32-NEXT: vmv.x.s t1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 18
-; RV32-NEXT: vmv.x.s a2, v12
+; RV32-NEXT: vmv.x.s t2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 19
-; RV32-NEXT: vmv.x.s a3, v12
+; RV32-NEXT: vmv.x.s t3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 20
-; RV32-NEXT: vmv.x.s a4, v12
+; RV32-NEXT: vmv.x.s t4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 21
-; RV32-NEXT: vmv.x.s a5, v12
+; RV32-NEXT: vmv.x.s t5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 22
-; RV32-NEXT: vmv.x.s a6, v12
+; RV32-NEXT: vmv.x.s t6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 23
-; RV32-NEXT: vmv.x.s a7, v12
+; RV32-NEXT: vmv.x.s s0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 24
-; RV32-NEXT: vmv.x.s t0, v12
+; RV32-NEXT: vmv.x.s s1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 25
-; RV32-NEXT: vmv.x.s t1, v12
+; RV32-NEXT: vmv.x.s s2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 26
-; RV32-NEXT: vmv.x.s t2, v12
+; RV32-NEXT: vmv.x.s s3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 27
-; RV32-NEXT: vmv.x.s t3, v12
+; RV32-NEXT: vmv.x.s s4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 28
-; RV32-NEXT: vmv.x.s t4, v12
+; RV32-NEXT: vmv.x.s s5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 29
-; RV32-NEXT: vmv.x.s t5, v12
+; RV32-NEXT: vmv.x.s s6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 30
-; RV32-NEXT: vmv.x.s t6, v12
+; RV32-NEXT: vmv.x.s s7, v12
; RV32-NEXT: vslidedown.vi v12, v10, 31
-; RV32-NEXT: vmv.x.s s0, v12
+; RV32-NEXT: vmv.x.s s8, v12
; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vrgather.vi v9, v10, 1
; RV32-NEXT: vmseq.vv v9, v8, v9
@@ -770,30 +823,6 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmseq.vv v12, v8, v13
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vrgather.vi v11, v10, 7
-; RV32-NEXT: vrgather.vi v13, v10, 8
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 9
-; RV32-NEXT: vrgather.vi v13, v10, 10
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 11
-; RV32-NEXT: vrgather.vi v13, v10, 12
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 13
-; RV32-NEXT: vrgather.vi v13, v10, 14
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 15
; RV32-NEXT: vmor.mm v9, v9, v12
; RV32-NEXT: vmseq.vv v10, v8, v11
; RV32-NEXT: vmseq.vx v11, v8, a0
@@ -826,55 +855,120 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vmseq.vx v11, v8, t6
; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s0
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s1
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s2
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s3
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s4
; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v8, v8, s0
+; RV32-NEXT: vmseq.vx v11, v8, s5
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s6
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s7
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v8, v8, s8
; RV32-NEXT: vmor.mm v8, v9, v8
; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 44(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s2, 36(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s5, 24(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s7, 16(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s8, 12(sp) # 4-byte Folded Reload
; RV32-NEXT: .cfi_restore s0
-; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_restore s1
+; RV32-NEXT: .cfi_restore s2
+; RV32-NEXT: .cfi_restore s3
+; RV32-NEXT: .cfi_restore s4
+; RV32-NEXT: .cfi_restore s5
+; RV32-NEXT: .cfi_restore s6
+; RV32-NEXT: .cfi_restore s7
+; RV32-NEXT: .cfi_restore s8
+; RV32-NEXT: addi sp, sp, 48
; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
;
; RV64-LABEL: match_v16i8_v32i8:
; RV64: # %bb.0:
-; RV64-NEXT: addi sp, sp, -16
-; RV64-NEXT: .cfi_def_cfa_offset 16
-; RV64-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: addi sp, sp, -80
+; RV64-NEXT: .cfi_def_cfa_offset 80
+; RV64-NEXT: sd s0, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s1, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s2, 56(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s3, 48(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s4, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s5, 32(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s6, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s7, 16(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s8, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset s0, -8
+; RV64-NEXT: .cfi_offset s1, -16
+; RV64-NEXT: .cfi_offset s2, -24
+; RV64-NEXT: .cfi_offset s3, -32
+; RV64-NEXT: .cfi_offset s4, -40
+; RV64-NEXT: .cfi_offset s5, -48
+; RV64-NEXT: .cfi_offset s6, -56
+; RV64-NEXT: .cfi_offset s7, -64
+; RV64-NEXT: .cfi_offset s8, -72
+; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v9, v10, 8
+; RV64-NEXT: vmv.x.s a0, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 9
+; RV64-NEXT: vmv.x.s a1, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 10
+; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 11
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 12
+; RV64-NEXT: vmv.x.s a4, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 13
+; RV64-NEXT: vmv.x.s a5, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 14
+; RV64-NEXT: vmv.x.s a6, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 15
+; RV64-NEXT: vmv.x.s a7, v9
; RV64-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV64-NEXT: vslidedown.vi v12, v10, 16
-; RV64-NEXT: vmv.x.s a0, v12
+; RV64-NEXT: vmv.x.s t0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 17
-; RV64-NEXT: vmv.x.s a1, v12
+; RV64-NEXT: vmv.x.s t1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 18
-; RV64-NEXT: vmv.x.s a2, v12
+; RV64-NEXT: vmv.x.s t2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 19
-; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: vmv.x.s t3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 20
-; RV64-NEXT: vmv.x.s a4, v12
+; RV64-NEXT: vmv.x.s t4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 21
-; RV64-NEXT: vmv.x.s a5, v12
+; RV64-NEXT: vmv.x.s t5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 22
-; RV64-NEXT: vmv.x.s a6, v12
+; RV64-NEXT: vmv.x.s t6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 23
-; RV64-NEXT: vmv.x.s a7, v12
+; RV64-NEXT: vmv.x.s s0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 24
-; RV64-NEXT: vmv.x.s t0, v12
+; RV64-NEXT: vmv.x.s s1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 25
-; RV64-NEXT: vmv.x.s t1, v12
+; RV64-NEXT: vmv.x.s s2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 26
-; RV64-NEXT: vmv.x.s t2, v12
+; RV64-NEXT: vmv.x.s s3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 27
-; RV64-NEXT: vmv.x.s t3, v12
+; RV64-NEXT: vmv.x.s s4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 28
-; RV64-NEXT: vmv.x.s t4, v12
+; RV64-NEXT: vmv.x.s s5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 29
-; RV64-NEXT: vmv.x.s t5, v12
+; RV64-NEXT: vmv.x.s s6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 30
-; RV64-NEXT: vmv.x.s t6, v12
+; RV64-NEXT: vmv.x.s s7, v12
; RV64-NEXT: vslidedown.vi v12, v10, 31
-; RV64-NEXT: vmv.x.s s0, v12
+; RV64-NEXT: vmv.x.s s8, v12
; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vrgather.vi v9, v10, 1
; RV64-NEXT: vmseq.vv v9, v8, v9
@@ -896,30 +990,6 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmseq.vv v12, v8, v13
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vrgather.vi v11, v10, 7
-; RV64-NEXT: vrgather.vi v13, v10, 8
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 9
-; RV64-NEXT: vrgather.vi v13, v10, 10
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 11
-; RV64-NEXT: vrgather.vi v13, v10, 12
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 13
-; RV64-NEXT: vrgather.vi v13, v10, 14
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 15
; RV64-NEXT: vmor.mm v9, v9, v12
; RV64-NEXT: vmseq.vv v10, v8, v11
; RV64-NEXT: vmseq.vx v11, v8, a0
@@ -952,13 +1022,45 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vmseq.vx v11, v8, t6
; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s0
; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v8, v8, s0
+; RV64-NEXT: vmseq.vx v11, v8, s1
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s2
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s3
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s4
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s5
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s6
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s7
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v8, v8, s8
; RV64-NEXT: vmor.mm v8, v9, v8
; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s1, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s2, 56(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s3, 48(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s4, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s5, 32(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s6, 24(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s7, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s8, 8(sp) # 8-byte Folded Reload
; RV64-NEXT: .cfi_restore s0
-; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_restore s1
+; RV64-NEXT: .cfi_restore s2
+; RV64-NEXT: .cfi_restore s3
+; RV64-NEXT: .cfi_restore s4
+; RV64-NEXT: .cfi_restore s5
+; RV64-NEXT: .cfi_restore s6
+; RV64-NEXT: .cfi_restore s7
+; RV64-NEXT: .cfi_restore s8
+; RV64-NEXT: addi sp, sp, 80
; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask)
@@ -987,16 +1089,46 @@ define <vscale x 4 x i1> @match_nxv4xi32_v4i32(<vscale x 4 x i32> %op1, <4 x i32
}
define <vscale x 2 x i1> @match_nxv2xi64_v2i64(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask) {
-; CHECK-LABEL: match_nxv2xi64_v2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT: vrgather.vi v12, v10, 1
-; CHECK-NEXT: vmseq.vv v14, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v10, 0
-; CHECK-NEXT: vmseq.vv v10, v8, v12
-; CHECK-NEXT: vmor.mm v8, v10, v14
-; CHECK-NEXT: vmand.mm v0, v8, v0
-; CHECK-NEXT: ret
+; RV32-LABEL: match_nxv2xi64_v2i64:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: li a0, 32
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vmv.x.s a1, v10
+; RV32-NEXT: vsrl.vx v11, v10, a0
+; RV32-NEXT: vmv.x.s a2, v11
+; RV32-NEXT: vslidedown.vi v10, v10, 1
+; RV32-NEXT: vmv.x.s a3, v10
+; RV32-NEXT: vsrl.vx v10, v10, a0
+; RV32-NEXT: vmv.x.s a0, v10
+; RV32-NEXT: sw a1, 8(sp)
+; RV32-NEXT: sw a2, 12(sp)
+; RV32-NEXT: sw a3, 0(sp)
+; RV32-NEXT: sw a0, 4(sp)
+; RV32-NEXT: addi a0, sp, 8
+; RV32-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV32-NEXT: vlse64.v v10, (a0), zero
+; RV32-NEXT: mv a0, sp
+; RV32-NEXT: vlse64.v v12, (a0), zero
+; RV32-NEXT: vmseq.vv v14, v8, v10
+; RV32-NEXT: vmseq.vv v10, v8, v12
+; RV32-NEXT: vmor.mm v8, v14, v10
+; RV32-NEXT: vmand.mm v0, v8, v0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: match_nxv2xi64_v2i64:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vrgather.vi v12, v10, 1
+; RV64-NEXT: vmseq.vv v14, v8, v12
+; RV64-NEXT: vrgather.vi v12, v10, 0
+; RV64-NEXT: vmseq.vv v10, v8, v12
+; RV64-NEXT: vmor.mm v8, v10, v14
+; RV64-NEXT: vmand.mm v0, v8, v0
+; RV64-NEXT: ret
%r = tail call <vscale x 2 x i1> @llvm.experimental.vector.match(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask)
ret <vscale x 2 x i1> %r
}
@@ -1005,16 +1137,18 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-LABEL: match_v4xi32_v4i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 2
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 3
+; CHECK-NEXT: vmv.x.s a1, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmor.mm v10, v11, v10
-; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v9, v8, v10
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v10, v9
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a1
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1023,16 +1157,47 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
}
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
-; CHECK-LABEL: match_v2xi64_v2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vmseq.vv v9, v8, v10
-; CHECK-NEXT: vmseq.vv v8, v8, v11
-; CHECK-NEXT: vmor.mm v8, v8, v9
-; CHECK-NEXT: vmand.mm v0, v8, v0
-; CHECK-NEXT: ret
+; RV32-LABEL: match_v2xi64_v2i64:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: li a0, 32
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vmv.x.s a1, v9
+; RV32-NEXT: vsrl.vx v10, v9, a0
+; RV32-NEXT: vmv.x.s a2, v10
+; RV32-NEXT: vslidedown.vi v9, v9, 1
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vsrl.vx v9, v9, a0
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: sw a1, 8(sp)
+; RV32-NEXT: sw a2, 12(sp)
+; RV32-NEXT: sw a3, 0(sp)
+; RV32-NEXT: sw a0, 4(sp)
+; RV32-NEXT: addi a0, sp, 8
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vlse64.v v9, (a0), zero
+; RV32-NEXT: mv a0, sp
+; RV32-NEXT: vlse64.v v10, (a0), zero
+; RV32-NEXT: vmseq.vv v9, v8, v9
+; RV32-NEXT: vmseq.vv v8, v8, v10
+; RV32-NEXT: vmor.mm v8, v9, v8
+; RV32-NEXT: vmand.mm v0, v8, v0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: match_v2xi64_v2i64:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v10, v9, 1
+; RV64-NEXT: vmv.x.s a0, v10
+; RV64-NEXT: vrgather.vi v10, v9, 0
+; RV64-NEXT: vmseq.vv v9, v8, v10
+; RV64-NEXT: vmseq.vx v8, v8, a0
+; RV64-NEXT: vmor.mm v8, v9, v8
+; RV64-NEXT: vmand.mm v0, v8, v0
+; RV64-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
}
>From f7a7b0b2b1abaea4b331f690524ad1b1f3050468 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 4 Aug 2026 10:03:32 +0000
Subject: [PATCH 3/3] Fixups
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 29 ++-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 -
.../AArch64/intrinsic-vector-match-sve2.ll | 36 ++--
.../RISCV/rvv/intrinsic-vector-match.ll | 193 +++++++-----------
7 files changed, 113 insertions(+), 160 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 7124553a1b4a8..43eba643bc110 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1631,7 +1631,9 @@ enum NodeType {
/// intrinsic.
/// Operands: Source, Needle, Mask
/// Source has the same number of elements as the result and Needle may have
- /// a different number of elements. The result type matches Mask.
+ /// a different number of elements. The result type matches Mask. The ISD
+ /// node supports result and mask types wider than i1, in these cases the
+ /// high bits conform to getBooleanContents similar to the SETCC operator.
VECTOR_MATCH,
/// The `llvm.loop.dependence.{war, raw}.mask` intrinsics
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 53395a1b1b690..b2a7b314525da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13852,18 +13852,33 @@ SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
SDValue Ret = DAG.getConstant(0, DL, CmpVT);
EVT NeedleEltVT = NeedleVT.getVectorElementType();
for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
- SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
- SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ SDValue Splat;
+ if (NeedleVT == SourceVT) {
+ // Prefer a shuffle over scalar extracts + splat for fixed vectors.
+ Splat = DAG.getVectorShuffle(
+ SourceVT, DL, Needle, DAG.getUNDEF(SourceVT),
+ SmallVector<int>(NeedleVT.getVectorNumElements(), I));
+ } else {
+ SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
+ Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ }
+
SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
}
- if (Mask.getValueType() != CmpVT)
- Mask = DAG.getBoolExtOrTrunc(Mask, DL, CmpVT, Mask.getValueType());
+ EVT UseVT = ResVT;
+ // If the result is immediately truncated, only extend to that type (to avoid
+ // unnecessary sign/zero extends).
+ if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
+ UseVT = N->user_begin()->getValueType(0);
+
+ Mask = DAG.getBoolExtOrTrunc(Mask, DL, UseVT, Mask.getValueType());
+ Ret = DAG.getBoolExtOrTrunc(Ret, DL, UseVT, Ret.getValueType());
- Ret = DAG.getNode(ISD::AND, DL, CmpVT, Ret, Mask);
- if (Ret.getValueType() != ResVT)
- Ret = DAG.getBoolExtOrTrunc(Ret, DL, ResVT, Ret.getValueType());
+ Ret = DAG.getNode(ISD::AND, DL, UseVT, Ret, Mask);
+ if (UseVT != ResVT)
+ Ret = DAG.getNode(ISD::ANY_EXTEND, DL, ResVT, Ret);
return Ret;
}
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index e5f2f3fc0e80e..5f2a29675919a 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1242,8 +1242,8 @@ void TargetLoweringBase::initActions() {
setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
Expand);
- // Only some target support this vector operation. Most need to expand it.
- setOperationAction(ISD::VECTOR_COMPRESS, VT, Expand);
+ // Only some target support these vector operation. Most need to expand it.
+ setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
// cttz.elts defaults to expand.
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index aebd304fccd2b..b041a2364a54d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1646,9 +1646,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
}
- for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
-
// FIXME: Move lowering for more nodes here if those are common between
// SVE and SME.
if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -1664,8 +1661,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
}
+
if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index aa5dd5f7cff4b..37505a5b187f8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1038,8 +1038,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (!isTypeLegal(VT))
continue;
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
-
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
// Mask VTs are custom-expanded into a series of standard nodes
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 1f4101c6b1b56..20aa26fef104a 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -18,11 +18,9 @@ define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
; CHECK-NEXT: umov w8, v1.b[0]
; CHECK-NEXT: dup v1.8b, w8
-; CHECK-NEXT: sbfx w8, w0, #0, #1
; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: umov w0, v0.b[0]
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: and w0, w8, w0
; CHECK-NEXT: ret
%r = tail call <1 x i1> @llvm.experimental.vector.match(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask)
ret <1 x i1> %r
@@ -104,11 +102,9 @@ define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mas
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -122,10 +118,8 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: shl v1.16b, v2.16b, #7
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -139,7 +133,6 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v4.16b, v1.b[0]
; CHECK-NEXT: dup v5.16b, v1.b[4]
; CHECK-NEXT: dup v1.16b, v1.b[6]
-; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
@@ -147,8 +140,7 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -354,14 +346,13 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s
; CHECK-NEXT: cmeq v5.4s, v0.4s, v5.4s
; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: shl v1.4h, v2.4h, #15
-; CHECK-NEXT: orr v2.16b, v4.16b, v3.16b
-; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: shl v1.4h, v2.4h, #15
; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <4 x i1> @llvm.experimental.vector.match(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
ret <4 x i1> %r
@@ -370,16 +361,15 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) #0 {
; CHECK-LABEL: match_v2xi64_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: shl v2.2s, v2.2s, #31
; CHECK-NEXT: dup v3.2d, v1.d[1]
; CHECK-NEXT: dup v1.2d, v1.d[0]
-; CHECK-NEXT: cmlt v2.2s, v2.2s, #0
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: shl v1.2s, v2.2s, #31
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: sshll v1.2d, v2.2s, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: cmlt v1.2s, v1.2s, #0
; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index d341093bd131f..78fd483a786b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -225,26 +225,10 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-LABEL: match_v16i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 8
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 9
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 10
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 11
-; CHECK-NEXT: vmv.x.s a3, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 12
-; CHECK-NEXT: vmv.x.s a4, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 13
-; CHECK-NEXT: vmv.x.s a5, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 14
-; CHECK-NEXT: vmv.x.s a6, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 15
-; CHECK-NEXT: vmv.x.s a7, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
@@ -261,24 +245,32 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-NEXT: vmseq.vv v12, v8, v13
; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vrgather.vi v11, v9, 7
+; CHECK-NEXT: vrgather.vi v13, v9, 8
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 9
+; CHECK-NEXT: vrgather.vi v13, v9, 10
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 11
+; CHECK-NEXT: vrgather.vi v13, v9, 12
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 13
+; CHECK-NEXT: vrgather.vi v13, v9, 14
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 15
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a3
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a4
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a5
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a6
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a7
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -290,32 +282,28 @@ define <8 x i1> @match_v8i8_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i8_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 5
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 6
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 7
-; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vrgather.vi v12, v9, 2
; CHECK-NEXT: vmor.mm v10, v11, v10
+; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 4
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 5
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 6
+; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 3
+; CHECK-NEXT: vrgather.vi v12, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v11
-; CHECK-NEXT: vmseq.vv v10, v8, v12
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a3
+; CHECK-NEXT: vmseq.vv v8, v8, v12
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -360,32 +348,28 @@ define <8 x i1> @match_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 5
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 6
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 7
-; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vrgather.vi v13, v9, 4
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 5
+; CHECK-NEXT: vrgather.vi v13, v9, 6
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a3
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1137,18 +1121,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-LABEL: match_v4xi32_v4i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 2
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 3
-; CHECK-NEXT: vmv.x.s a1, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vmseq.vv v9, v8, v10
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v10, v9
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a1
+; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmor.mm v10, v11, v10
+; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vmor.mm v9, v10, v12
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1157,47 +1139,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
}
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
-; RV32-LABEL: match_v2xi64_v2i64:
-; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: li a0, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vmv.x.s a1, v9
-; RV32-NEXT: vsrl.vx v10, v9, a0
-; RV32-NEXT: vmv.x.s a2, v10
-; RV32-NEXT: vslidedown.vi v9, v9, 1
-; RV32-NEXT: vmv.x.s a3, v9
-; RV32-NEXT: vsrl.vx v9, v9, a0
-; RV32-NEXT: vmv.x.s a0, v9
-; RV32-NEXT: sw a1, 8(sp)
-; RV32-NEXT: sw a2, 12(sp)
-; RV32-NEXT: sw a3, 0(sp)
-; RV32-NEXT: sw a0, 4(sp)
-; RV32-NEXT: addi a0, sp, 8
-; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV32-NEXT: vlse64.v v9, (a0), zero
-; RV32-NEXT: mv a0, sp
-; RV32-NEXT: vlse64.v v10, (a0), zero
-; RV32-NEXT: vmseq.vv v9, v8, v9
-; RV32-NEXT: vmseq.vv v8, v8, v10
-; RV32-NEXT: vmor.mm v8, v9, v8
-; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: addi sp, sp, 16
-; RV32-NEXT: .cfi_def_cfa_offset 0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: match_v2xi64_v2i64:
-; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v10, v9, 1
-; RV64-NEXT: vmv.x.s a0, v10
-; RV64-NEXT: vrgather.vi v10, v9, 0
-; RV64-NEXT: vmseq.vv v9, v8, v10
-; RV64-NEXT: vmseq.vx v8, v8, a0
-; RV64-NEXT: vmor.mm v8, v9, v8
-; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ret
+; CHECK-LABEL: match_v2xi64_v2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vrgather.vi v11, v9, 0
+; CHECK-NEXT: vmseq.vv v9, v8, v10
+; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmor.mm v8, v8, v9
+; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
}
More information about the llvm-commits
mailing list