[llvm] [SDAG] Add basic type legalization for experimental.vector.match (PR #213344)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 01:28:45 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/213344
>From fa7fa5d127b7be841752597c93b7494614eb5470 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 31 Jul 2026 12:11:35 +0000
Subject: [PATCH 1/8] [SDAG] Add basic type legalization for
experimental.vector.match
This adds a new ISD::VECTOR_MATCH opcode and implements basic type
legalization. Currently, this operation is only used on AArch64 (as the
only target with any form of match instruction).
Right now, this does not support many more cases than the existing
lowering. A follow up patch will handle supporting needle sizes <= the
maximum size without falling back to expansion.
Assisted-by: Codex
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 8 +-
llvm/include/llvm/CodeGen/ISDOpcodes.h | 7 +
llvm/include/llvm/CodeGen/TargetLowering.h | 14 +-
.../include/llvm/Target/TargetSelectionDAG.td | 7 +
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 22 ++
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 8 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 137 +++++++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 3 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 31 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 80 ++---
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 3 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 36 ++-
.../AArch64/intrinsic-vector-match-sve2.ll | 278 +++++-------------
15 files changed, 367 insertions(+), 294 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..6c7bb56563ab6 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2572,14 +2572,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
unsigned SearchSize = NeedleTy->getNumElements();
- // If we're not expanding the intrinsic then we assume this is cheap to
- // implement.
- EVT SearchVT = getTLI()->getValueType(DL, SearchTy);
- if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize))
- return getTypeLegalizationCost(RetTy).first;
-
// Approximate the cost based on the expansion code in
- // SelectionDAGBuilder.
+ // TargetLowering::expandVectorMatch.
InstructionCost Cost = 0;
Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, NeedleTy,
CostKind, 1, nullptr, nullptr);
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 23ef5d22963ae..7124553a1b4a8 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1627,6 +1627,13 @@ enum NodeType {
/// bits conform to getBooleanContents similar to the SETCC operator.
GET_ACTIVE_LANE_MASK,
+ /// VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match
+ /// intrinsic.
+ /// Operands: Source, Needle, Mask
+ /// Source has the same number of elements as the result and Needle may have
+ /// a different number of elements. The result type matches Mask.
+ VECTOR_MATCH,
+
/// The `llvm.loop.dependence.{war, raw}.mask` intrinsics
/// Operands: Load pointer, Store pointer, Element size, Lane offset
/// Output: Mask
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9a525d69b3ee8..e77b6cf059c95 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -515,6 +515,8 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
+ virtual bool shouldExpandVectorMatch() const { return true; }
+
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
@@ -530,13 +532,6 @@ class LLVM_ABI TargetLoweringBase {
bool ZeroIsPoison,
const ConstantRange *VScaleRange) const;
- /// Return true if the @llvm.experimental.vector.match intrinsic should be
- /// expanded for vector type `VT' and search size `SearchSize' using generic
- /// code in SelectionDAGBuilder.
- virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const {
- return true;
- }
-
// Return true if op(vecreduce(x), vecreduce(y)) should be reassociated to
// vecreduce(op(x, y)) for the reduction opcode RedOpc.
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const {
@@ -5773,6 +5768,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// \returns The expansion result or SDValue() if it fails.
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand VECTOR_MATCH nodes.
+ /// \param N Node to expand
+ /// \returns The expansion result or SDValue() if it fails.
+ SDValue expandVectorMatch(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand VECTOR_FIND_LAST_ACTIVE nodes
/// \param N Node to expand
/// \returns The expansion result or SDValue() if it fails.
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 37b7b8716bcd8..07b9e435981c1 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -936,6 +936,13 @@ def get_active_lane_mask
SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<1, 2>]>,
[]>;
+def vector_match
+ : SDNode<"ISD::VECTOR_MATCH",
+ SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>,
+ SDTCisVec<3>, SDTCisSameNumEltsAs<0, 1>,
+ SDTCisSameAs<0, 3>]>,
+ []>;
+
// Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
// these internally. Don't reference these directly.
def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 8fbd44918173f..fe6aabc984907 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4647,6 +4647,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VP_CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandVPCTTZElements(Node, DAG));
break;
+ case ISD::VECTOR_MATCH:
+ Results.push_back(TLI.expandVectorMatch(Node, DAG));
+ break;
case ISD::CLEAR_CACHE:
// The default expansion of llvm.clear_cache is simply a no-op for those
// targets where it is not needed.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 2fee0b280ed9b..59b2098ea9bac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -168,6 +168,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = PromoteIntRes_VECTOR_MATCH(N);
+ break;
case ISD::PARTIAL_REDUCE_UMLA:
case ISD::PARTIAL_REDUCE_SMLA:
@@ -2293,6 +2296,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = PromoteIntOp_VECTOR_MATCH(N, OpNo);
+ break;
case ISD::MASKED_UDIV:
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
@@ -3166,6 +3172,16 @@ SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ assert(OpNo < 3 && "Unexpected operand for promotion");
+ if (OpNo != 2)
+ return TLI.expandVectorMatch(N, DAG);
+
+ SmallVector<SDValue, 3> NewOps(N->ops());
+ NewOps[2] = PromoteTargetBoolean(N->getOperand(2), N->getValueType(0));
+ return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
assert(OpNo == 2);
SmallVector<SDValue, 3> NewOps(N->ops());
@@ -6605,6 +6621,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
}
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_MATCH(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getNode(ISD::VECTOR_MATCH, SDLoc(N), NVT, N->ops());
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
SDLoc DL(N);
EVT VT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 91a3c44e49738..fe5b116ab0800 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -367,6 +367,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_READ_REGISTER(SDNode *N);
SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_MATCH(SDNode *N);
SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N);
@@ -425,6 +426,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_UnaryBooleanVectorOp(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N);
SDValue PromoteIntOp_MaskedBinOp(SDNode *N, unsigned OpNo);
@@ -896,6 +898,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecOp_FAKE_USE(SDNode *N);
SDValue ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
SDValue ScalarizeVecOp_CTTZ_ELTS(SDNode *N);
+ SDValue ScalarizeVecRes_VECTOR_MATCH(SDNode *N);
+ SDValue ScalarizeVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
SDValue ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
@@ -976,6 +980,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo, SDValue &Hi);
// Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
bool SplitVectorOperand(SDNode *N, unsigned OpNo);
@@ -1012,6 +1017,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_VECTOR_HISTOGRAM(SDNode *N);
SDValue SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
// Vector Widening Support: LegalizeVectorTypes.cpp
@@ -1080,6 +1086,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);
SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue WidenVecRes_VECTOR_MATCH(SDNode *N);
void WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N);
void WidenVecRes_VECTOR_INTERLEAVE(SDNode *N);
@@ -1138,6 +1145,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecOp_CttzElements(SDNode *N);
SDValue WidenVecOp_VP_CttzElements(SDNode *N);
SDValue WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo);
/// Helper function to generate a set of operations to perform
/// a vector operation for a wider type.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 19b221bb98d46..d7809e24beb23 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -93,6 +93,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
+ case ISD::VECTOR_MATCH:
+ R = ScalarizeVecRes_VECTOR_MATCH(N);
+ break;
case ISD::POISON:
case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
@@ -981,6 +984,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::CTTZ_ELTS_ZERO_POISON:
Res = ScalarizeVecOp_CTTZ_ELTS(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
+ break;
case ISD::MASKED_UDIV:
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
@@ -1320,6 +1326,20 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
return DAG.getZExtOrTrunc(SetCC, SDLoc(N), N->getValueType(0));
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
+ SDLoc DL(N);
+ // Reuse the expansion (which should scalarize).
+ SDValue Mask = TLI.expandVectorMatch(N, DAG);
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
+ N->getValueType(0).getScalarType(), Mask,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
+ unsigned OpNo) {
+ return TLI.expandVectorMatch(N, DAG);
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
assert(OpNo == 2 && "Can only scalarize mask operand");
SDLoc DL(N);
@@ -1668,6 +1688,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
break;
+ case ISD::VECTOR_MATCH:
+ SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
+ break;
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -3754,6 +3777,20 @@ void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
Hi = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, HiVT, HiStartVal, Op1);
}
+void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ SDValue SourceLo, SourceHi;
+ GetSplitVector(N->getOperand(0), SourceLo, SourceHi);
+ SDValue MaskLo, MaskHi;
+ GetSplitVector(N->getOperand(2), MaskLo, MaskHi);
+ SDLoc DL(N);
+
+ Lo = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
+ N->getOperand(1), MaskLo, N->getFlags());
+ Hi = DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
+ N->getOperand(1), MaskHi, N->getFlags());
+}
+
void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
unsigned Factor = N->getNumOperands();
@@ -3998,6 +4035,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::PARTIAL_REDUCE_FMLA:
Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
+ break;
}
// If the result is null, the sub-method took care of registering results etc.
@@ -5208,6 +5248,44 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
MMO, IndexType);
}
+SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ SDLoc DL(N);
+
+ if (OpNo != 1) {
+ auto SplitOperand = [&](SDValue Op, SDValue &Lo, SDValue &Hi) {
+ if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSplitVector)
+ GetSplitVector(Op, Lo, Hi);
+ else
+ std::tie(Lo, Hi) = DAG.SplitVector(Op, DL);
+ };
+
+ SDValue SourceLo, SourceHi;
+ SplitOperand(N->getOperand(0), SourceLo, SourceHi);
+ SDValue MaskLo, MaskHi;
+ SplitOperand(N->getOperand(2), MaskLo, MaskHi);
+
+ SDValue MatchLo =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
+ N->getOperand(1), MaskLo, N->getFlags());
+ SDValue MatchHi =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
+ N->getOperand(1), MaskHi, N->getFlags());
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), MatchLo,
+ MatchHi);
+ }
+
+ SDValue NeedleLo, NeedleHi;
+ GetSplitVector(N->getOperand(1), NeedleLo, NeedleHi);
+
+ SDValue MatchLo =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
+ NeedleLo, N->getOperand(2), N->getFlags());
+ SDValue MatchHi =
+ DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0), N->getOperand(0),
+ NeedleHi, N->getOperand(2), N->getFlags());
+ return DAG.getNode(ISD::OR, DL, N->getValueType(0), MatchLo, MatchHi);
+}
+
SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
SDValue Acc = N->getOperand(0);
assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
@@ -5353,6 +5431,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_INTERLEAVE:
WidenVecRes_VECTOR_INTERLEAVE(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = WidenVecRes_VECTOR_MATCH(N);
+ break;
case ISD::VECTOR_DEINTERLEAVE:
WidenVecRes_VECTOR_DEINTERLEAVE(N);
break;
@@ -7552,6 +7633,22 @@ void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
}
}
+SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
+ SDLoc DL(N);
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+ EVT SourceVT = N->getOperand(0).getValueType();
+ EVT WideSourceVT =
+ EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
+ WidenVT.getVectorElementCount());
+
+ SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
+ N->getOperand(0), 0);
+ SDValue WideMask = DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, WidenVT),
+ N->getOperand(2), 0);
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
+ N->getOperand(1), WideMask, N->getFlags());
+}
+
void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
EVT VT = N->getValueType(0);
EVT EltVT = VT.getVectorElementType();
@@ -7824,6 +7921,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_FIND_LAST_ACTIVE:
Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
break;
+ case ISD::VECTOR_MATCH:
+ Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
+ break;
}
// If Res is null, the sub-method took care of registering the result.
@@ -8850,6 +8950,43 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
WideMask);
}
+SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
+ if (OpNo != 1) {
+ SDLoc DL(N);
+ EVT ResVT = N->getValueType(0);
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
+ EVT SourceVT = N->getOperand(0).getValueType();
+ EVT WideSourceVT =
+ EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
+ WidenVT.getVectorElementCount());
+
+ SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
+ N->getOperand(0), 0);
+ SDValue WideMask = DAG.getInsertSubvector(
+ DL, DAG.getConstant(0, DL, WidenVT), N->getOperand(2), 0);
+ SDValue WideMatch = DAG.getNode(ISD::VECTOR_MATCH, DL, WidenVT, WideSource,
+ N->getOperand(1), WideMask, N->getFlags());
+ return DAG.getExtractSubvector(DL, ResVT, WideMatch, 0);
+ }
+
+ SDLoc DL(N);
+ SDValue Needle = N->getOperand(1);
+ EVT NeedleVT = Needle.getValueType();
+ if (NeedleVT.getVectorNumElements() == 1)
+ return TLI.expandVectorMatch(N, DAG);
+
+ EVT WidenNeedleVT = TLI.getTypeToTransformTo(*DAG.getContext(), NeedleVT);
+
+ SDValue Fill =
+ DAG.getExtractVectorElt(DL, NeedleVT.getVectorElementType(), Needle, 0);
+ SDValue WideNeedle = DAG.getSplatVector(WidenNeedleVT, DL, Fill);
+ WideNeedle = DAG.getInsertSubvector(DL, WideNeedle, Needle, 0);
+
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0),
+ N->getOperand(0), WideNeedle, N->getOperand(2),
+ N->getFlags());
+}
+
//===----------------------------------------------------------------------===//
// Vector Widening Utilities
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a2b9a14ed9543..dd07267d2f074 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8532,30 +8532,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));
SDValue Mask = getValue(I.getOperand(2));
- EVT Op1VT = Op1.getValueType();
- EVT Op2VT = Op2.getValueType();
EVT ResVT = Mask.getValueType();
- unsigned SearchSize = Op2VT.getVectorNumElements();
-
- // If the target has native support for this vector match operation, lower
- // the intrinsic untouched; otherwise, expand it below.
- if (!TLI.shouldExpandVectorMatch(Op1VT, SearchSize)) {
- visitTargetIntrinsic(I, Intrinsic);
+ if (!TLI.shouldExpandVectorMatch()) {
+ setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
return;
}
- SDValue Ret = DAG.getConstant(0, sdl, ResVT);
-
- for (unsigned i = 0; i < SearchSize; ++i) {
- SDValue Op2Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl,
- Op2VT.getVectorElementType(), Op2,
- DAG.getVectorIdxConstant(i, sdl));
- SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, sdl, Op1VT, Op2Elem);
- SDValue Cmp = DAG.getSetCC(sdl, ResVT, Op1, Splat, ISD::SETEQ);
- Ret = DAG.getNode(ISD::OR, sdl, ResVT, Ret, Cmp);
- }
-
- setValue(&I, DAG.getNode(ISD::AND, sdl, ResVT, Ret, Mask));
+ SDValue Match = DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask);
+ setValue(&I, TLI.expandVectorMatch(Match.getNode(), DAG));
return;
}
case Intrinsic::vector_reverse:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 028f0b9486c01..5c1b280697262 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -610,6 +610,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::GET_ACTIVE_LANE_MASK:
return "get_active_lane_mask";
+ case ISD::VECTOR_MATCH:
+ return "vector_match";
+
case ISD::PARTIAL_REDUCE_UMLA:
return "partial_reduce_umla";
case ISD::PARTIAL_REDUCE_SMLA:
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 61238db24e5c0..12c33a066915b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13864,6 +13864,37 @@ SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
return DAG.getZExtOrTrunc(Sub, DL, VT);
}
+SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
+ SDLoc DL(N);
+ SDValue Source = N->getOperand(0);
+ SDValue Needle = N->getOperand(1);
+ SDValue Mask = N->getOperand(2);
+ EVT SourceVT = Source.getValueType();
+ EVT NeedleVT = Needle.getValueType();
+ EVT ResVT = N->getValueType(0);
+ EVT CmpVT =
+ getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SourceVT);
+
+ assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
+
+ SDValue Ret = DAG.getConstant(0, DL, CmpVT);
+ EVT NeedleEltVT = NeedleVT.getVectorElementType();
+ for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
+ SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
+ SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
+ Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
+ }
+
+ if (Mask.getValueType() != CmpVT)
+ Mask = DAG.getBoolExtOrTrunc(Mask, DL, CmpVT, Mask.getValueType());
+
+ Ret = DAG.getNode(ISD::AND, DL, CmpVT, Ret, Mask);
+ if (Ret.getValueType() != ResVT)
+ Ret = DAG.getBoolExtOrTrunc(Ret, DL, ResVT, Ret.getValueType());
+ return Ret;
+}
+
SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
SelectionDAG &DAG) const {
SDLoc DL(N);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2ac6c5fbc471a..5fc1f4426a16d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1613,6 +1613,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
}
+ for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+
// FIXME: Move lowering for more nodes here if those are common between
// SVE and SME.
if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -1628,6 +1631,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+ }
+ if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
+ for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
+
+ for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
+ setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
}
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
@@ -2448,6 +2459,8 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
return false;
}
+bool AArch64TargetLowering::shouldExpandVectorMatch() const { return false; }
+
bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
if (!Subtarget->isSVEorStreamingSVEAvailable())
return true;
@@ -2459,19 +2472,6 @@ bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
VT != MVT::v4i1 && VT != MVT::v2i1;
}
-bool AArch64TargetLowering::shouldExpandVectorMatch(EVT VT,
- unsigned SearchSize) const {
- // MATCH is SVE2 and only available in non-streaming mode.
- if (!Subtarget->hasSVE2() || !Subtarget->isSVEAvailable())
- return true;
- // Furthermore, we can only use it for 8-bit or 16-bit elements.
- if (VT == MVT::nxv8i16 || VT == MVT::v8i16)
- return SearchSize != 8;
- if (VT == MVT::nxv16i8 || VT == MVT::v16i8 || VT == MVT::v8i8)
- return SearchSize != 8 && SearchSize != 16;
- return true;
-}
-
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
@@ -6459,21 +6459,29 @@ static SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
static SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
SDLoc DL(Op);
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
-
- auto Op1 = Op.getOperand(1);
- auto Op2 = Op.getOperand(2);
- auto Mask = Op.getOperand(3);
+ auto Op1 = Op.getOperand(0);
+ auto Op2 = Op.getOperand(1);
+ auto Mask = Op.getOperand(2);
EVT Op1VT = Op1.getValueType();
EVT Op2VT = Op2.getValueType();
EVT ResVT = Op.getValueType();
+ unsigned SearchSize = Op2VT.getVectorNumElements();
assert((Op1VT.getVectorElementType() == MVT::i8 ||
Op1VT.getVectorElementType() == MVT::i16) &&
"Expected 8-bit or 16-bit characters.");
+ if ((Op1VT == MVT::nxv8i16 || Op1VT == MVT::v8i16) && SearchSize != 8)
+ return SDValue();
+
+ if ((Op1VT == MVT::nxv16i8 || Op1VT == MVT::v16i8 || Op1VT == MVT::v8i8) &&
+ SearchSize != 8 && SearchSize != 16)
+ return SDValue();
+
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
+
// Scalable vector type used to wrap operands.
// A single container is enough for both operands because ultimately the
// operands will have to be wrapped to the same type (nxv16i8 or nxv8i16).
@@ -7200,9 +7208,6 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
ADDLV, DAG.getConstant(0, DL, MVT::i64));
return EXTRACT_VEC_ELT;
}
- case Intrinsic::experimental_vector_match: {
- return LowerVectorMatch(Op, DAG);
- }
case Intrinsic::aarch64_cls:
case Intrinsic::aarch64_cls64: {
SDValue Res = DAG.getNode(ISD::CTLS, DL, Op.getOperand(1).getValueType(),
@@ -8836,6 +8841,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_INTERLEAVE(Op, DAG);
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
+ case ISD::VECTOR_MATCH:
+ return LowerVectorMatch(Op, DAG, Subtarget);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
@@ -32334,6 +32341,21 @@ void AArch64TargetLowering::ReplaceNodeResults(
case ISD::GET_ACTIVE_LANE_MASK:
ReplaceGetActiveLaneMaskResults(N, Results, DAG);
return;
+ case ISD::VECTOR_MATCH: {
+ EVT VT = N->getValueType(0);
+ if (!VT.isFixedLengthVectorOf(MVT::i1))
+ return;
+
+ // NOTE: Only trivial type promotion is supported.
+ EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
+ if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
+ return;
+
+ SDLoc DL(N);
+ SDValue V = DAG.getNode(ISD::VECTOR_MATCH, DL, NewVT, N->ops());
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
+ return;
+ }
case ISD::INTRINSIC_WO_CHAIN: {
EVT VT = N->getValueType(0);
@@ -32390,20 +32412,6 @@ void AArch64TargetLowering::ReplaceNodeResults(
DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, RuntimePStateSM));
return;
}
- case Intrinsic::experimental_vector_match: {
- if (!VT.isFixedLengthVectorOf(MVT::i1))
- return;
-
- // NOTE: Only trivial type promotion is supported.
- EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
- if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
- return;
-
- SDLoc DL(N);
- auto V = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, NewVT, N->ops());
- Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
- return;
- }
}
}
case ISD::READ_REGISTER: {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 66a6261d2a991..68b4d2fd7c6f6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -565,11 +565,10 @@ class AArch64TargetLowering : public TargetLowering {
bool AllowUnknown = false) const override;
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
+ bool shouldExpandVectorMatch() const override;
bool shouldExpandCttzElements(EVT VT) const override;
- bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const override;
-
/// If a change in streaming mode is required on entry to/return from a
/// function call it emits and returns the corresponding SMSTART or SMSTOP
/// node. \p Condition should be one of the enum values from
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b55e0856c792e..5248dbda9f064 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1149,18 +1149,30 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
auto *NeedleTy = cast<FixedVectorType>(ICA.getArgTypes()[1]);
EVT SearchVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
unsigned SearchSize = NeedleTy->getNumElements();
- if (!getTLI()->shouldExpandVectorMatch(SearchVT, SearchSize)) {
- // Base cost for MATCH instructions. At least on the Neoverse V2 and
- // Neoverse V3, these are cheap operations with the same latency as a
- // vector ADD. In most cases, however, we also need to do an extra DUP.
- // For fixed-length vectors we currently need an extra five--six
- // instructions besides the MATCH.
- InstructionCost Cost = 4;
- if (isa<FixedVectorType>(RetTy))
- Cost += 10;
- return Cost;
- }
- break;
+ auto IsSupportedTypeAndSearchSize = [&]() {
+ if (SearchVT == MVT::nxv8i16 || SearchVT == MVT::v8i16)
+ return SearchSize == 8;
+
+ if (SearchVT == MVT::nxv16i8 || SearchVT == MVT::v16i8 ||
+ SearchVT == MVT::v8i8)
+ return SearchSize == 8 || SearchSize == 16;
+
+ return false;
+ };
+
+ if (!ST->hasSVE2() || !ST->isSVEAvailable() ||
+ !IsSupportedTypeAndSearchSize())
+ break;
+
+ // Base cost for MATCH instructions. At least on the Neoverse V2 and
+ // Neoverse V3, these are cheap operations with the same latency as a
+ // vector ADD. In most cases, however, we also need to do an extra DUP.
+ // For fixed-length vectors we currently need an extra five--six
+ // instructions besides the MATCH.
+ InstructionCost Cost = 4;
+ if (isa<FixedVectorType>(RetTy))
+ Cost += 10;
+ return Cost;
}
case Intrinsic::cttz: {
auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 8ef3314ba2681..1f4101c6b1b56 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -12,6 +12,22 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
ret <vscale x 16 x i1> %r
}
+define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #0 {
+; CHECK-LABEL: match_v1i8_v1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: umov w8, v1.b[0]
+; CHECK-NEXT: dup v1.8b, w8
+; CHECK-NEXT: sbfx w8, w0, #0, #1
+; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: umov w0, v0.b[0]
+; CHECK-NEXT: ret
+ %r = tail call <1 x i1> @llvm.experimental.vector.match(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask)
+ ret <1 x i1> %r
+}
+
define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v2i8:
; CHECK: // %bb.0:
@@ -88,9 +104,11 @@ define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mas
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -104,8 +122,10 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: shl v1.16b, v2.16b, #7
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -119,6 +139,7 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v4.16b, v1.b[0]
; CHECK-NEXT: dup v5.16b, v1.b[4]
; CHECK-NEXT: dup v1.16b, v1.b[6]
+; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
@@ -126,7 +147,8 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -231,112 +253,13 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v32i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
-; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov z3.b, z1.b[1]
-; CHECK-NEXT: mov z4.b, b1
; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: mov z5.b, z1.b[2]
-; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z3.b, z1.b[3]
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z5.b
-; CHECK-NEXT: mov z4.b, z1.b[4]
-; CHECK-NEXT: mov p2.b, p3/m, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[5]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[6]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[7]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[8]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[9]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[10]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[11]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[12]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z1.b[13]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: mov z4.b, z1.b[14]
-; CHECK-NEXT: mov z1.b, z1.b[15]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, b2
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z4.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[1]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[2]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[3]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[4]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[5]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[6]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[7]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[8]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[9]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[10]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[11]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[12]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[13]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z3.b, z2.b[14]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, z2.b[15]
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: cmpeq p1.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p3.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
-; CHECK-NEXT: orr p0.b, p0/z, p2.b, p1.b
-; CHECK-NEXT: addvl sp, sp, #1
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: mov z1.q, q1
+; CHECK-NEXT: match p1.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: match p2.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: sel p0.b, p2, p2.b, p1.b
; CHECK-NEXT: ret
%r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <32 x i8> %op2, <vscale x 16 x i1> %mask)
ret <vscale x 16 x i1> %r
@@ -345,107 +268,34 @@ define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8
define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask) #0 {
; CHECK-LABEL: match_v16i8_v32i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: dup v4.16b, v1.b[1]
-; CHECK-NEXT: dup v5.16b, v1.b[0]
-; CHECK-NEXT: dup v6.16b, v1.b[2]
-; CHECK-NEXT: dup v7.16b, v1.b[3]
-; CHECK-NEXT: dup v16.16b, v1.b[4]
-; CHECK-NEXT: dup v17.16b, v1.b[5]
-; CHECK-NEXT: dup v18.16b, v1.b[6]
-; CHECK-NEXT: dup v19.16b, v1.b[7]
-; CHECK-NEXT: dup v20.16b, v1.b[8]
-; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
-; CHECK-NEXT: cmeq v6.16b, v0.16b, v6.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v7.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v16.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v17.16b
-; CHECK-NEXT: dup v21.16b, v2.b[7]
-; CHECK-NEXT: dup v22.16b, v1.b[10]
-; CHECK-NEXT: orr v4.16b, v5.16b, v4.16b
-; CHECK-NEXT: orr v5.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v18.16b, v1.b[9]
-; CHECK-NEXT: dup v19.16b, v1.b[11]
-; CHECK-NEXT: dup v20.16b, v1.b[12]
-; CHECK-NEXT: cmeq v22.16b, v0.16b, v22.16b
-; CHECK-NEXT: orr v4.16b, v4.16b, v5.16b
-; CHECK-NEXT: orr v5.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: dup v18.16b, v1.b[13]
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v19.16b, v2.b[0]
-; CHECK-NEXT: dup v20.16b, v2.b[1]
-; CHECK-NEXT: orr v4.16b, v4.16b, v5.16b
-; CHECK-NEXT: dup v5.16b, v2.b[6]
-; CHECK-NEXT: orr v6.16b, v6.16b, v7.16b
-; CHECK-NEXT: orr v7.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v19.16b, v2.b[2]
-; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[8]
-; CHECK-NEXT: orr v6.16b, v6.16b, v22.16b
-; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT: dup v16.16b, v1.b[14]
-; CHECK-NEXT: dup v1.16b, v1.b[15]
-; CHECK-NEXT: orr v17.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v19.16b, v2.b[3]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[9]
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v16.16b
-; CHECK-NEXT: cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT: orr v4.16b, v4.16b, v6.16b
-; CHECK-NEXT: orr v17.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v18.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v19.16b, v2.b[4]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v20.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v21.16b, v2.b[10]
-; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b
-; CHECK-NEXT: orr v16.16b, v17.16b, v18.16b
-; CHECK-NEXT: cmeq v17.16b, v0.16b, v19.16b
-; CHECK-NEXT: dup v18.16b, v2.b[5]
-; CHECK-NEXT: orr v5.16b, v5.16b, v20.16b
-; CHECK-NEXT: cmeq v19.16b, v0.16b, v21.16b
-; CHECK-NEXT: dup v20.16b, v2.b[11]
-; CHECK-NEXT: orr v1.16b, v7.16b, v1.16b
-; CHECK-NEXT: orr v6.16b, v16.16b, v17.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: dup v17.16b, v2.b[12]
-; CHECK-NEXT: orr v5.16b, v5.16b, v19.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v20.16b
-; CHECK-NEXT: dup v18.16b, v2.b[13]
-; CHECK-NEXT: dup v19.16b, v2.b[14]
-; CHECK-NEXT: orr v1.16b, v4.16b, v1.16b
-; CHECK-NEXT: dup v2.16b, v2.b[15]
-; CHECK-NEXT: orr v4.16b, v6.16b, v7.16b
-; CHECK-NEXT: cmeq v6.16b, v0.16b, v17.16b
-; CHECK-NEXT: orr v5.16b, v5.16b, v16.16b
-; CHECK-NEXT: cmeq v7.16b, v0.16b, v18.16b
-; CHECK-NEXT: cmeq v16.16b, v0.16b, v19.16b
-; CHECK-NEXT: cmeq v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v4.16b, v5.16b, v6.16b
-; CHECK-NEXT: orr v5.16b, v7.16b, v16.16b
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
+; CHECK-NEXT: shl v3.16b, v3.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: cmpne p1.b, p0/z, z3.b, #0
+; CHECK-NEXT: match p0.b, p1/z, z0.b, z2.b
+; CHECK-NEXT: match p2.b, p1/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v3.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
}
+define <vscale x 32 x i1> @match_nxv32i8_v16i8(<vscale x 32 x i8> %op1, <16 x i8> %op2, <vscale x 32 x i1> %mask) #0 {
+; CHECK-LABEL: match_nxv32i8_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: mov z2.q, q2
+; CHECK-NEXT: match p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: match p1.b, p1/z, z1.b, z2.b
+; CHECK-NEXT: ret
+ %r = tail call <vscale x 32 x i1> @llvm.experimental.vector.match(<vscale x 32 x i8> %op1, <16 x i8> %op2, <vscale x 32 x i1> %mask)
+ ret <vscale x 32 x i1> %r
+}
+
; Data types not supported by MATCH.
; Note: The cases for SVE could be made tighter.
@@ -502,12 +352,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-NEXT: dup v1.4s, v1.s[3]
; CHECK-NEXT: cmeq v3.4s, v0.4s, v3.4s
; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b
-; CHECK-NEXT: cmeq v4.4s, v0.4s, v5.4s
+; CHECK-NEXT: cmeq v5.4s, v0.4s, v5.4s
; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v3.16b, v3.16b, v4.16b
-; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s
-; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: shl v1.4h, v2.4h, #15
+; CHECK-NEXT: orr v2.16b, v4.16b, v3.16b
+; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
%r = tail call <4 x i1> @llvm.experimental.vector.match(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
ret <4 x i1> %r
@@ -516,12 +370,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) #0 {
; CHECK-LABEL: match_v2xi64_v2i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: shl v2.2s, v2.2s, #31
; CHECK-NEXT: dup v3.2d, v1.d[1]
; CHECK-NEXT: dup v1.2d, v1.d[0]
+; CHECK-NEXT: cmlt v2.2s, v2.2s, #0
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: addhn v0.2s, v0.2d, v3.2d
-; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: sshll v1.2d, v2.2s, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
>From 70246e5e7c3b1d5158c1aaacd6e4f2ab2765944f Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 3 Aug 2026 08:58:00 +0000
Subject: [PATCH 2/8] Fixups
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 -
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 -
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 -
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 -
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +
.../RISCV/rvv/intrinsic-vector-match.ll | 555 ++++++++++++------
8 files changed, 367 insertions(+), 211 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index e77b6cf059c95..46b52c30cded6 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -515,8 +515,6 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
- virtual bool shouldExpandVectorMatch() const { return true; }
-
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index fe6aabc984907..8fbd44918173f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4647,9 +4647,6 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VP_CTTZ_ELTS_ZERO_POISON:
Results.push_back(TLI.expandVPCTTZElements(Node, DAG));
break;
- case ISD::VECTOR_MATCH:
- Results.push_back(TLI.expandVectorMatch(Node, DAG));
- break;
case ISD::CLEAR_CACHE:
// The default expansion of llvm.clear_cache is simply a no-op for those
// targets where it is not needed.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7b97c41ac2dbc..44fc8c0d41172 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -489,6 +489,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
case ISD::MASKED_SREM:
+ case ISD::VECTOR_MATCH:
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
case ISD::SMULFIX:
@@ -1324,6 +1325,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
+ case ISD::VECTOR_MATCH:
+ Results.push_back(TLI.expandVectorMatch(Node, DAG));
+ return;
case ISD::SREM:
case ISD::UREM:
ExpandREM(Node, Results);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index dd07267d2f074..6261f2a7f0437 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8533,13 +8533,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Op2 = getValue(I.getOperand(1));
SDValue Mask = getValue(I.getOperand(2));
EVT ResVT = Mask.getValueType();
- if (!TLI.shouldExpandVectorMatch()) {
- setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
- return;
- }
-
- SDValue Match = DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask);
- setValue(&I, TLI.expandVectorMatch(Match.getNode(), DAG));
+ setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
return;
}
case Intrinsic::vector_reverse:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 5fc1f4426a16d..615c4122cb62f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2459,8 +2459,6 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
return false;
}
-bool AArch64TargetLowering::shouldExpandVectorMatch() const { return false; }
-
bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
if (!Subtarget->isSVEorStreamingSVEAvailable())
return true;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 68b4d2fd7c6f6..b4952430958c7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -565,8 +565,6 @@ class AArch64TargetLowering : public TargetLowering {
bool AllowUnknown = false) const override;
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
- bool shouldExpandVectorMatch() const override;
-
bool shouldExpandCttzElements(EVT VT) const override;
/// If a change in streaming mode is required on entry to/return from a
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ea4803e59ebc1..2c8e7f0201bd8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1044,6 +1044,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (!isTypeLegal(VT))
continue;
+ setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
+
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
// Mask VTs are custom-expanded into a series of standard nodes
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index 6febcd22e3288..d341093bd131f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -225,10 +225,26 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-LABEL: match_v16i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 8
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 9
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 10
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 11
+; CHECK-NEXT: vmv.x.s a3, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vmv.x.s a4, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vmv.x.s a5, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vmv.x.s a7, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
@@ -245,32 +261,24 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-NEXT: vmseq.vv v12, v8, v13
; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vrgather.vi v11, v9, 7
-; CHECK-NEXT: vrgather.vi v13, v9, 8
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 9
-; CHECK-NEXT: vrgather.vi v13, v9, 10
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 11
-; CHECK-NEXT: vrgather.vi v13, v9, 12
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 13
-; CHECK-NEXT: vrgather.vi v13, v9, 14
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 15
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a3
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a4
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a5
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a6
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a7
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -282,28 +290,32 @@ define <8 x i1> @match_v8i8_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i8_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
+; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vrgather.vi v12, v9, 2
; CHECK-NEXT: vmor.mm v10, v11, v10
-; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 5
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 7
+; CHECK-NEXT: vrgather.vi v12, v9, 3
; CHECK-NEXT: vmor.mm v9, v10, v11
-; CHECK-NEXT: vmseq.vv v8, v8, v12
+; CHECK-NEXT: vmseq.vv v10, v8, v12
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -348,28 +360,32 @@ define <8 x i1> @match_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
+; CHECK-NEXT: vmv.x.s a1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
+; CHECK-NEXT: vmv.x.s a2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
+; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vrgather.vi v13, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 5
-; CHECK-NEXT: vrgather.vi v13, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v12
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v13
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vrgather.vi v11, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -383,22 +399,30 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-LABEL: match_v8i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 8
+; CHECK-NEXT: vslidedown.vi v10, v9, 4
; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 9
+; CHECK-NEXT: vslidedown.vi v10, v9, 5
; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 10
+; CHECK-NEXT: vslidedown.vi v10, v9, 6
; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 11
+; CHECK-NEXT: vslidedown.vi v10, v9, 7
; CHECK-NEXT: vmv.x.s a3, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vslidedown.vi v10, v9, 8
; CHECK-NEXT: vmv.x.s a4, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vslidedown.vi v10, v9, 9
; CHECK-NEXT: vmv.x.s a5, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vslidedown.vi v10, v9, 10
; CHECK-NEXT: vmv.x.s a6, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vslidedown.vi v10, v9, 11
; CHECK-NEXT: vmv.x.s a7, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 12
+; CHECK-NEXT: vmv.x.s t0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 13
+; CHECK-NEXT: vmv.x.s t1, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 14
+; CHECK-NEXT: vmv.x.s t2, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 15
+; CHECK-NEXT: vmv.x.s t3, v10
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
@@ -408,18 +432,6 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vmseq.vv v11, v8, v12
; CHECK-NEXT: vrgather.vi v12, v9, 3
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 4
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 5
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 6
-; CHECK-NEXT: vmor.mm v10, v10, v11
-; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v11
; CHECK-NEXT: vmseq.vv v10, v8, v12
; CHECK-NEXT: vmseq.vx v11, v8, a0
@@ -436,8 +448,16 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v9, v9, v11
; CHECK-NEXT: vmseq.vx v11, v8, a6
; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, a7
; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a7
+; CHECK-NEXT: vmseq.vx v11, v8, t0
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmseq.vx v10, v8, t1
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v11, v8, t2
+; CHECK-NEXT: vmor.mm v9, v9, v10
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, t3
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -712,43 +732,76 @@ define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8
define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask) {
; RV32-LABEL: match_v16i8_v32i8:
; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: addi sp, sp, -48
+; RV32-NEXT: .cfi_def_cfa_offset 48
+; RV32-NEXT: sw s0, 44(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s1, 40(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s3, 32(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s5, 24(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s6, 20(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s7, 16(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s8, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset s0, -4
+; RV32-NEXT: .cfi_offset s1, -8
+; RV32-NEXT: .cfi_offset s2, -12
+; RV32-NEXT: .cfi_offset s3, -16
+; RV32-NEXT: .cfi_offset s4, -20
+; RV32-NEXT: .cfi_offset s5, -24
+; RV32-NEXT: .cfi_offset s6, -28
+; RV32-NEXT: .cfi_offset s7, -32
+; RV32-NEXT: .cfi_offset s8, -36
+; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma
+; RV32-NEXT: vslidedown.vi v9, v10, 8
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 9
+; RV32-NEXT: vmv.x.s a1, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 10
+; RV32-NEXT: vmv.x.s a2, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 11
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 12
+; RV32-NEXT: vmv.x.s a4, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 13
+; RV32-NEXT: vmv.x.s a5, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 14
+; RV32-NEXT: vmv.x.s a6, v9
+; RV32-NEXT: vslidedown.vi v9, v10, 15
+; RV32-NEXT: vmv.x.s a7, v9
; RV32-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV32-NEXT: vslidedown.vi v12, v10, 16
-; RV32-NEXT: vmv.x.s a0, v12
+; RV32-NEXT: vmv.x.s t0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 17
-; RV32-NEXT: vmv.x.s a1, v12
+; RV32-NEXT: vmv.x.s t1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 18
-; RV32-NEXT: vmv.x.s a2, v12
+; RV32-NEXT: vmv.x.s t2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 19
-; RV32-NEXT: vmv.x.s a3, v12
+; RV32-NEXT: vmv.x.s t3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 20
-; RV32-NEXT: vmv.x.s a4, v12
+; RV32-NEXT: vmv.x.s t4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 21
-; RV32-NEXT: vmv.x.s a5, v12
+; RV32-NEXT: vmv.x.s t5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 22
-; RV32-NEXT: vmv.x.s a6, v12
+; RV32-NEXT: vmv.x.s t6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 23
-; RV32-NEXT: vmv.x.s a7, v12
+; RV32-NEXT: vmv.x.s s0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 24
-; RV32-NEXT: vmv.x.s t0, v12
+; RV32-NEXT: vmv.x.s s1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 25
-; RV32-NEXT: vmv.x.s t1, v12
+; RV32-NEXT: vmv.x.s s2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 26
-; RV32-NEXT: vmv.x.s t2, v12
+; RV32-NEXT: vmv.x.s s3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 27
-; RV32-NEXT: vmv.x.s t3, v12
+; RV32-NEXT: vmv.x.s s4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 28
-; RV32-NEXT: vmv.x.s t4, v12
+; RV32-NEXT: vmv.x.s s5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 29
-; RV32-NEXT: vmv.x.s t5, v12
+; RV32-NEXT: vmv.x.s s6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 30
-; RV32-NEXT: vmv.x.s t6, v12
+; RV32-NEXT: vmv.x.s s7, v12
; RV32-NEXT: vslidedown.vi v12, v10, 31
-; RV32-NEXT: vmv.x.s s0, v12
+; RV32-NEXT: vmv.x.s s8, v12
; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vrgather.vi v9, v10, 1
; RV32-NEXT: vmseq.vv v9, v8, v9
@@ -770,30 +823,6 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmseq.vv v12, v8, v13
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vrgather.vi v11, v10, 7
-; RV32-NEXT: vrgather.vi v13, v10, 8
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 9
-; RV32-NEXT: vrgather.vi v13, v10, 10
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 11
-; RV32-NEXT: vrgather.vi v13, v10, 12
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 13
-; RV32-NEXT: vrgather.vi v13, v10, 14
-; RV32-NEXT: vmor.mm v9, v9, v12
-; RV32-NEXT: vmseq.vv v11, v8, v11
-; RV32-NEXT: vmseq.vv v12, v8, v13
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vrgather.vi v11, v10, 15
; RV32-NEXT: vmor.mm v9, v9, v12
; RV32-NEXT: vmseq.vv v10, v8, v11
; RV32-NEXT: vmseq.vx v11, v8, a0
@@ -826,55 +855,120 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vmseq.vx v11, v8, t6
; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s0
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s1
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s2
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s3
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s4
; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v8, v8, s0
+; RV32-NEXT: vmseq.vx v11, v8, s5
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmseq.vx v10, v8, s6
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v11, v8, s7
+; RV32-NEXT: vmor.mm v9, v9, v10
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vmseq.vx v8, v8, s8
; RV32-NEXT: vmor.mm v8, v9, v8
; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 44(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s2, 36(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s5, 24(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s7, 16(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s8, 12(sp) # 4-byte Folded Reload
; RV32-NEXT: .cfi_restore s0
-; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_restore s1
+; RV32-NEXT: .cfi_restore s2
+; RV32-NEXT: .cfi_restore s3
+; RV32-NEXT: .cfi_restore s4
+; RV32-NEXT: .cfi_restore s5
+; RV32-NEXT: .cfi_restore s6
+; RV32-NEXT: .cfi_restore s7
+; RV32-NEXT: .cfi_restore s8
+; RV32-NEXT: addi sp, sp, 48
; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
;
; RV64-LABEL: match_v16i8_v32i8:
; RV64: # %bb.0:
-; RV64-NEXT: addi sp, sp, -16
-; RV64-NEXT: .cfi_def_cfa_offset 16
-; RV64-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: addi sp, sp, -80
+; RV64-NEXT: .cfi_def_cfa_offset 80
+; RV64-NEXT: sd s0, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s1, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s2, 56(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s3, 48(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s4, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s5, 32(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s6, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s7, 16(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s8, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset s0, -8
+; RV64-NEXT: .cfi_offset s1, -16
+; RV64-NEXT: .cfi_offset s2, -24
+; RV64-NEXT: .cfi_offset s3, -32
+; RV64-NEXT: .cfi_offset s4, -40
+; RV64-NEXT: .cfi_offset s5, -48
+; RV64-NEXT: .cfi_offset s6, -56
+; RV64-NEXT: .cfi_offset s7, -64
+; RV64-NEXT: .cfi_offset s8, -72
+; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v9, v10, 8
+; RV64-NEXT: vmv.x.s a0, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 9
+; RV64-NEXT: vmv.x.s a1, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 10
+; RV64-NEXT: vmv.x.s a2, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 11
+; RV64-NEXT: vmv.x.s a3, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 12
+; RV64-NEXT: vmv.x.s a4, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 13
+; RV64-NEXT: vmv.x.s a5, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 14
+; RV64-NEXT: vmv.x.s a6, v9
+; RV64-NEXT: vslidedown.vi v9, v10, 15
+; RV64-NEXT: vmv.x.s a7, v9
; RV64-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV64-NEXT: vslidedown.vi v12, v10, 16
-; RV64-NEXT: vmv.x.s a0, v12
+; RV64-NEXT: vmv.x.s t0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 17
-; RV64-NEXT: vmv.x.s a1, v12
+; RV64-NEXT: vmv.x.s t1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 18
-; RV64-NEXT: vmv.x.s a2, v12
+; RV64-NEXT: vmv.x.s t2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 19
-; RV64-NEXT: vmv.x.s a3, v12
+; RV64-NEXT: vmv.x.s t3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 20
-; RV64-NEXT: vmv.x.s a4, v12
+; RV64-NEXT: vmv.x.s t4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 21
-; RV64-NEXT: vmv.x.s a5, v12
+; RV64-NEXT: vmv.x.s t5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 22
-; RV64-NEXT: vmv.x.s a6, v12
+; RV64-NEXT: vmv.x.s t6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 23
-; RV64-NEXT: vmv.x.s a7, v12
+; RV64-NEXT: vmv.x.s s0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 24
-; RV64-NEXT: vmv.x.s t0, v12
+; RV64-NEXT: vmv.x.s s1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 25
-; RV64-NEXT: vmv.x.s t1, v12
+; RV64-NEXT: vmv.x.s s2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 26
-; RV64-NEXT: vmv.x.s t2, v12
+; RV64-NEXT: vmv.x.s s3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 27
-; RV64-NEXT: vmv.x.s t3, v12
+; RV64-NEXT: vmv.x.s s4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 28
-; RV64-NEXT: vmv.x.s t4, v12
+; RV64-NEXT: vmv.x.s s5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 29
-; RV64-NEXT: vmv.x.s t5, v12
+; RV64-NEXT: vmv.x.s s6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 30
-; RV64-NEXT: vmv.x.s t6, v12
+; RV64-NEXT: vmv.x.s s7, v12
; RV64-NEXT: vslidedown.vi v12, v10, 31
-; RV64-NEXT: vmv.x.s s0, v12
+; RV64-NEXT: vmv.x.s s8, v12
; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vrgather.vi v9, v10, 1
; RV64-NEXT: vmseq.vv v9, v8, v9
@@ -896,30 +990,6 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmseq.vv v12, v8, v13
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vrgather.vi v11, v10, 7
-; RV64-NEXT: vrgather.vi v13, v10, 8
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 9
-; RV64-NEXT: vrgather.vi v13, v10, 10
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 11
-; RV64-NEXT: vrgather.vi v13, v10, 12
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 13
-; RV64-NEXT: vrgather.vi v13, v10, 14
-; RV64-NEXT: vmor.mm v9, v9, v12
-; RV64-NEXT: vmseq.vv v11, v8, v11
-; RV64-NEXT: vmseq.vv v12, v8, v13
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vrgather.vi v11, v10, 15
; RV64-NEXT: vmor.mm v9, v9, v12
; RV64-NEXT: vmseq.vv v10, v8, v11
; RV64-NEXT: vmseq.vx v11, v8, a0
@@ -952,13 +1022,45 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vmseq.vx v11, v8, t6
; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s0
; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v8, v8, s0
+; RV64-NEXT: vmseq.vx v11, v8, s1
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s2
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s3
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s4
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s5
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmseq.vx v10, v8, s6
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v11, v8, s7
+; RV64-NEXT: vmor.mm v9, v9, v10
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vmseq.vx v8, v8, s8
; RV64-NEXT: vmor.mm v8, v9, v8
; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s1, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s2, 56(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s3, 48(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s4, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s5, 32(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s6, 24(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s7, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s8, 8(sp) # 8-byte Folded Reload
; RV64-NEXT: .cfi_restore s0
-; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: .cfi_restore s1
+; RV64-NEXT: .cfi_restore s2
+; RV64-NEXT: .cfi_restore s3
+; RV64-NEXT: .cfi_restore s4
+; RV64-NEXT: .cfi_restore s5
+; RV64-NEXT: .cfi_restore s6
+; RV64-NEXT: .cfi_restore s7
+; RV64-NEXT: .cfi_restore s8
+; RV64-NEXT: addi sp, sp, 80
; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask)
@@ -987,16 +1089,46 @@ define <vscale x 4 x i1> @match_nxv4xi32_v4i32(<vscale x 4 x i32> %op1, <4 x i32
}
define <vscale x 2 x i1> @match_nxv2xi64_v2i64(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask) {
-; CHECK-LABEL: match_nxv2xi64_v2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT: vrgather.vi v12, v10, 1
-; CHECK-NEXT: vmseq.vv v14, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v10, 0
-; CHECK-NEXT: vmseq.vv v10, v8, v12
-; CHECK-NEXT: vmor.mm v8, v10, v14
-; CHECK-NEXT: vmand.mm v0, v8, v0
-; CHECK-NEXT: ret
+; RV32-LABEL: match_nxv2xi64_v2i64:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: li a0, 32
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vmv.x.s a1, v10
+; RV32-NEXT: vsrl.vx v11, v10, a0
+; RV32-NEXT: vmv.x.s a2, v11
+; RV32-NEXT: vslidedown.vi v10, v10, 1
+; RV32-NEXT: vmv.x.s a3, v10
+; RV32-NEXT: vsrl.vx v10, v10, a0
+; RV32-NEXT: vmv.x.s a0, v10
+; RV32-NEXT: sw a1, 8(sp)
+; RV32-NEXT: sw a2, 12(sp)
+; RV32-NEXT: sw a3, 0(sp)
+; RV32-NEXT: sw a0, 4(sp)
+; RV32-NEXT: addi a0, sp, 8
+; RV32-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV32-NEXT: vlse64.v v10, (a0), zero
+; RV32-NEXT: mv a0, sp
+; RV32-NEXT: vlse64.v v12, (a0), zero
+; RV32-NEXT: vmseq.vv v14, v8, v10
+; RV32-NEXT: vmseq.vv v10, v8, v12
+; RV32-NEXT: vmor.mm v8, v14, v10
+; RV32-NEXT: vmand.mm v0, v8, v0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: match_nxv2xi64_v2i64:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT: vrgather.vi v12, v10, 1
+; RV64-NEXT: vmseq.vv v14, v8, v12
+; RV64-NEXT: vrgather.vi v12, v10, 0
+; RV64-NEXT: vmseq.vv v10, v8, v12
+; RV64-NEXT: vmor.mm v8, v10, v14
+; RV64-NEXT: vmand.mm v0, v8, v0
+; RV64-NEXT: ret
%r = tail call <vscale x 2 x i1> @llvm.experimental.vector.match(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask)
ret <vscale x 2 x i1> %r
}
@@ -1005,16 +1137,18 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-LABEL: match_v4xi32_v4i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v10, v9, 2
+; CHECK-NEXT: vmv.x.s a0, v10
+; CHECK-NEXT: vslidedown.vi v10, v9, 3
+; CHECK-NEXT: vmv.x.s a1, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vrgather.vi v12, v9, 2
-; CHECK-NEXT: vmseq.vv v10, v8, v10
-; CHECK-NEXT: vmseq.vv v11, v8, v11
-; CHECK-NEXT: vmseq.vv v12, v8, v12
-; CHECK-NEXT: vmor.mm v10, v11, v10
-; CHECK-NEXT: vrgather.vi v11, v9, 3
-; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmseq.vv v9, v8, v10
+; CHECK-NEXT: vmseq.vv v10, v8, v11
+; CHECK-NEXT: vmseq.vx v11, v8, a0
+; CHECK-NEXT: vmor.mm v9, v10, v9
+; CHECK-NEXT: vmor.mm v9, v9, v11
+; CHECK-NEXT: vmseq.vx v8, v8, a1
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1023,16 +1157,47 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
}
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
-; CHECK-LABEL: match_v2xi64_v2i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vmseq.vv v9, v8, v10
-; CHECK-NEXT: vmseq.vv v8, v8, v11
-; CHECK-NEXT: vmor.mm v8, v8, v9
-; CHECK-NEXT: vmand.mm v0, v8, v0
-; CHECK-NEXT: ret
+; RV32-LABEL: match_v2xi64_v2i64:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: li a0, 32
+; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT: vmv.x.s a1, v9
+; RV32-NEXT: vsrl.vx v10, v9, a0
+; RV32-NEXT: vmv.x.s a2, v10
+; RV32-NEXT: vslidedown.vi v9, v9, 1
+; RV32-NEXT: vmv.x.s a3, v9
+; RV32-NEXT: vsrl.vx v9, v9, a0
+; RV32-NEXT: vmv.x.s a0, v9
+; RV32-NEXT: sw a1, 8(sp)
+; RV32-NEXT: sw a2, 12(sp)
+; RV32-NEXT: sw a3, 0(sp)
+; RV32-NEXT: sw a0, 4(sp)
+; RV32-NEXT: addi a0, sp, 8
+; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT: vlse64.v v9, (a0), zero
+; RV32-NEXT: mv a0, sp
+; RV32-NEXT: vlse64.v v10, (a0), zero
+; RV32-NEXT: vmseq.vv v9, v8, v9
+; RV32-NEXT: vmseq.vv v8, v8, v10
+; RV32-NEXT: vmor.mm v8, v9, v8
+; RV32-NEXT: vmand.mm v0, v8, v0
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: match_v2xi64_v2i64:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vslidedown.vi v10, v9, 1
+; RV64-NEXT: vmv.x.s a0, v10
+; RV64-NEXT: vrgather.vi v10, v9, 0
+; RV64-NEXT: vmseq.vv v9, v8, v10
+; RV64-NEXT: vmseq.vx v8, v8, a0
+; RV64-NEXT: vmor.mm v8, v9, v8
+; RV64-NEXT: vmand.mm v0, v8, v0
+; RV64-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
}
>From 6e9a3eefe5acb16ac73a6f5d60ff6f8db706df53 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 4 Aug 2026 10:03:32 +0000
Subject: [PATCH 3/8] Fixups
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 29 ++-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 -
.../AArch64/intrinsic-vector-match-sve2.ll | 36 ++--
.../RISCV/rvv/intrinsic-vector-match.ll | 193 +++++++-----------
7 files changed, 113 insertions(+), 160 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 7124553a1b4a8..43eba643bc110 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1631,7 +1631,9 @@ enum NodeType {
/// intrinsic.
/// Operands: Source, Needle, Mask
/// Source has the same number of elements as the result and Needle may have
- /// a different number of elements. The result type matches Mask.
+ /// a different number of elements. The result type matches Mask. The ISD
+ /// node supports result and mask types wider than i1, in these cases the
+ /// high bits conform to getBooleanContents similar to the SETCC operator.
VECTOR_MATCH,
/// The `llvm.loop.dependence.{war, raw}.mask` intrinsics
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 12c33a066915b..c1ab43d35ae17 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13880,18 +13880,33 @@ SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
SDValue Ret = DAG.getConstant(0, DL, CmpVT);
EVT NeedleEltVT = NeedleVT.getVectorElementType();
for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
- SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
- SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ SDValue Splat;
+ if (NeedleVT == SourceVT) {
+ // Prefer a shuffle over scalar extracts + splat for fixed vectors.
+ Splat = DAG.getVectorShuffle(
+ SourceVT, DL, Needle, DAG.getUNDEF(SourceVT),
+ SmallVector<int>(NeedleVT.getVectorNumElements(), I));
+ } else {
+ SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
+ Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
+ }
+
SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
}
- if (Mask.getValueType() != CmpVT)
- Mask = DAG.getBoolExtOrTrunc(Mask, DL, CmpVT, Mask.getValueType());
+ EVT UseVT = ResVT;
+ // If the result is immediately truncated, only extend to that type (to avoid
+ // unnecessary sign/zero extends).
+ if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
+ UseVT = N->user_begin()->getValueType(0);
+
+ Mask = DAG.getBoolExtOrTrunc(Mask, DL, UseVT, Mask.getValueType());
+ Ret = DAG.getBoolExtOrTrunc(Ret, DL, UseVT, Ret.getValueType());
- Ret = DAG.getNode(ISD::AND, DL, CmpVT, Ret, Mask);
- if (Ret.getValueType() != ResVT)
- Ret = DAG.getBoolExtOrTrunc(Ret, DL, ResVT, Ret.getValueType());
+ Ret = DAG.getNode(ISD::AND, DL, UseVT, Ret, Mask);
+ if (UseVT != ResVT)
+ Ret = DAG.getNode(ISD::ANY_EXTEND, DL, ResVT, Ret);
return Ret;
}
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 051ba4670be40..668731ae003a2 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1173,8 +1173,8 @@ void TargetLoweringBase::initActions() {
setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
Expand);
- // Only some target support this vector operation. Most need to expand it.
- setOperationAction(ISD::VECTOR_COMPRESS, VT, Expand);
+ // Only some target support these vector operation. Most need to expand it.
+ setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
// cttz.elts defaults to expand.
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 615c4122cb62f..cead5b3c406f7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1613,9 +1613,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
}
- for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
-
// FIXME: Move lowering for more nodes here if those are common between
// SVE and SME.
if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -1631,8 +1628,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
}
+
if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 2c8e7f0201bd8..ea4803e59ebc1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1044,8 +1044,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (!isTypeLegal(VT))
continue;
- setOperationAction(ISD::VECTOR_MATCH, VT, Expand);
-
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
// Mask VTs are custom-expanded into a series of standard nodes
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 1f4101c6b1b56..20aa26fef104a 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -18,11 +18,9 @@ define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
; CHECK-NEXT: umov w8, v1.b[0]
; CHECK-NEXT: dup v1.8b, w8
-; CHECK-NEXT: sbfx w8, w0, #0, #1
; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: fmov s1, w8
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: umov w0, v0.b[0]
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: and w0, w8, w0
; CHECK-NEXT: ret
%r = tail call <1 x i1> @llvm.experimental.vector.match(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask)
ret <1 x i1> %r
@@ -104,11 +102,9 @@ define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mas
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -122,10 +118,8 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v1.16b, v1.b[0]
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: shl v1.16b, v2.16b, #7
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -139,7 +133,6 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: dup v4.16b, v1.b[0]
; CHECK-NEXT: dup v5.16b, v1.b[4]
; CHECK-NEXT: dup v1.16b, v1.b[6]
-; CHECK-NEXT: shl v2.16b, v2.16b, #7
; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
@@ -147,8 +140,7 @@ define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mas
; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: cmlt v1.16b, v2.16b, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -354,14 +346,13 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s
; CHECK-NEXT: cmeq v5.4s, v0.4s, v5.4s
; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: shl v1.4h, v2.4h, #15
-; CHECK-NEXT: orr v2.16b, v4.16b, v3.16b
-; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: shl v1.4h, v2.4h, #15
; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <4 x i1> @llvm.experimental.vector.match(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
ret <4 x i1> %r
@@ -370,16 +361,15 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) #0 {
; CHECK-LABEL: match_v2xi64_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: shl v2.2s, v2.2s, #31
; CHECK-NEXT: dup v3.2d, v1.d[1]
; CHECK-NEXT: dup v1.2d, v1.d[0]
-; CHECK-NEXT: cmlt v2.2s, v2.2s, #0
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: shl v1.2s, v2.2s, #31
; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: sshll v1.2d, v2.2s, #0
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: cmlt v1.2s, v1.2s, #0
; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index d341093bd131f..78fd483a786b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -225,26 +225,10 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-LABEL: match_v16i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 8
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 9
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 10
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 11
-; CHECK-NEXT: vmv.x.s a3, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 12
-; CHECK-NEXT: vmv.x.s a4, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 13
-; CHECK-NEXT: vmv.x.s a5, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 14
-; CHECK-NEXT: vmv.x.s a6, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 15
-; CHECK-NEXT: vmv.x.s a7, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
@@ -261,24 +245,32 @@ define <16 x i1> @match_v16i8_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %m
; CHECK-NEXT: vmseq.vv v12, v8, v13
; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vrgather.vi v11, v9, 7
+; CHECK-NEXT: vrgather.vi v13, v9, 8
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 9
+; CHECK-NEXT: vrgather.vi v13, v9, 10
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 11
+; CHECK-NEXT: vrgather.vi v13, v9, 12
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 13
+; CHECK-NEXT: vrgather.vi v13, v9, 14
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 15
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a3
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a4
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a5
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a6
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a7
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -290,32 +282,28 @@ define <8 x i1> @match_v8i8_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i8_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 5
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 6
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 7
-; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vrgather.vi v12, v9, 2
; CHECK-NEXT: vmor.mm v10, v11, v10
+; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 4
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 5
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 6
+; CHECK-NEXT: vmor.mm v10, v10, v11
; CHECK-NEXT: vmseq.vv v11, v8, v12
-; CHECK-NEXT: vrgather.vi v12, v9, 3
+; CHECK-NEXT: vrgather.vi v12, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v11
-; CHECK-NEXT: vmseq.vv v10, v8, v12
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a3
+; CHECK-NEXT: vmseq.vv v8, v8, v12
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -360,32 +348,28 @@ define <8 x i1> @match_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
; CHECK-LABEL: match_v8i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 5
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 6
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 7
-; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
-; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vrgather.vi v11, v9, 0
; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
; CHECK-NEXT: vmseq.vv v11, v8, v11
; CHECK-NEXT: vmseq.vv v12, v8, v12
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vrgather.vi v13, v9, 4
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 5
+; CHECK-NEXT: vrgather.vi v13, v9, 6
+; CHECK-NEXT: vmor.mm v10, v10, v12
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v13
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vrgather.vi v11, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v12
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a2
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a3
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1137,18 +1121,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-LABEL: match_v4xi32_v4i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 2
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 3
-; CHECK-NEXT: vmv.x.s a1, v10
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
-; CHECK-NEXT: vmseq.vv v9, v8, v10
-; CHECK-NEXT: vmseq.vv v10, v8, v11
-; CHECK-NEXT: vmseq.vx v11, v8, a0
-; CHECK-NEXT: vmor.mm v9, v10, v9
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, a1
+; CHECK-NEXT: vrgather.vi v12, v9, 2
+; CHECK-NEXT: vmseq.vv v10, v8, v10
+; CHECK-NEXT: vmseq.vv v11, v8, v11
+; CHECK-NEXT: vmseq.vv v12, v8, v12
+; CHECK-NEXT: vmor.mm v10, v11, v10
+; CHECK-NEXT: vrgather.vi v11, v9, 3
+; CHECK-NEXT: vmor.mm v9, v10, v12
+; CHECK-NEXT: vmseq.vv v8, v8, v11
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -1157,47 +1139,16 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
}
define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
-; RV32-LABEL: match_v2xi64_v2i64:
-; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: li a0, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vmv.x.s a1, v9
-; RV32-NEXT: vsrl.vx v10, v9, a0
-; RV32-NEXT: vmv.x.s a2, v10
-; RV32-NEXT: vslidedown.vi v9, v9, 1
-; RV32-NEXT: vmv.x.s a3, v9
-; RV32-NEXT: vsrl.vx v9, v9, a0
-; RV32-NEXT: vmv.x.s a0, v9
-; RV32-NEXT: sw a1, 8(sp)
-; RV32-NEXT: sw a2, 12(sp)
-; RV32-NEXT: sw a3, 0(sp)
-; RV32-NEXT: sw a0, 4(sp)
-; RV32-NEXT: addi a0, sp, 8
-; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV32-NEXT: vlse64.v v9, (a0), zero
-; RV32-NEXT: mv a0, sp
-; RV32-NEXT: vlse64.v v10, (a0), zero
-; RV32-NEXT: vmseq.vv v9, v8, v9
-; RV32-NEXT: vmseq.vv v8, v8, v10
-; RV32-NEXT: vmor.mm v8, v9, v8
-; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: addi sp, sp, 16
-; RV32-NEXT: .cfi_def_cfa_offset 0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: match_v2xi64_v2i64:
-; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v10, v9, 1
-; RV64-NEXT: vmv.x.s a0, v10
-; RV64-NEXT: vrgather.vi v10, v9, 0
-; RV64-NEXT: vmseq.vv v9, v8, v10
-; RV64-NEXT: vmseq.vx v8, v8, a0
-; RV64-NEXT: vmor.mm v8, v9, v8
-; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ret
+; CHECK-LABEL: match_v2xi64_v2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT: vrgather.vi v10, v9, 1
+; CHECK-NEXT: vrgather.vi v11, v9, 0
+; CHECK-NEXT: vmseq.vv v9, v8, v10
+; CHECK-NEXT: vmseq.vv v8, v8, v11
+; CHECK-NEXT: vmor.mm v8, v8, v9
+; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
ret <2 x i1> %r
}
>From 4b9c7d0377666947f2b6a119b7b81712a61c174f Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 4 Aug 2026 12:18:45 +0000
Subject: [PATCH 4/8] Fixups
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cead5b3c406f7..7a3aa51d0bc07 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8837,7 +8837,7 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::GET_ACTIVE_LANE_MASK:
return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
case ISD::VECTOR_MATCH:
- return LowerVectorMatch(Op, DAG, Subtarget);
+ return LowerVectorMatch(Op, DAG);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
>From 05ff2b3e4da1321f7212e836173795f96e277375 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 4 Aug 2026 12:39:00 +0000
Subject: [PATCH 5/8] Fixups
---
.../AArch64/intrinsic-vector-match-sve2.ll | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 20aa26fef104a..82912df0c8fee 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -15,9 +15,6 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #0 {
; CHECK-LABEL: match_v1i8_v1i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: umov w8, v1.b[0]
-; CHECK-NEXT: dup v1.8b, w8
; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
; CHECK-NEXT: umov w8, v0.b[0]
; CHECK-NEXT: and w0, w8, w0
@@ -344,14 +341,13 @@ define <4 x i1> @match_v4xi32_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %ma
; CHECK-NEXT: dup v1.4s, v1.s[3]
; CHECK-NEXT: cmeq v3.4s, v0.4s, v3.4s
; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT: cmeq v5.4s, v0.4s, v5.4s
+; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b
+; CHECK-NEXT: cmeq v4.4s, v0.4s, v5.4s
; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
-; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
-; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT: shl v1.4h, v2.4h, #15
-; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: orr v3.16b, v3.16b, v4.16b
; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <4 x i1> @llvm.experimental.vector.match(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask)
@@ -366,9 +362,8 @@ define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %ma
; CHECK-NEXT: cmeq v3.2d, v0.2d, v3.2d
; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d
; CHECK-NEXT: shl v1.2s, v2.2s, #31
-; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: addhn v0.2s, v0.2d, v3.2d
; CHECK-NEXT: cmlt v1.2s, v1.2s, #0
-; CHECK-NEXT: xtn v0.2s, v0.2d
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%r = tail call <2 x i1> @llvm.experimental.vector.match(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask)
>From ab88626af52452b09d0749b17ee7e7cb850a444b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 4 Aug 2026 18:49:32 +0000
Subject: [PATCH 6/8] Fixups
---
llvm/lib/CodeGen/TargetLoweringBase.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 668731ae003a2..71eda1048bc11 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1173,7 +1173,7 @@ void TargetLoweringBase::initActions() {
setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
Expand);
- // Only some target support these vector operation. Most need to expand it.
+ // Only some target support these vector operations. Default them to Expand.
setOperationAction({ISD::VECTOR_COMPRESS, ISD::VECTOR_MATCH}, VT, Expand);
// cttz.elts defaults to expand.
>From 268b1ac2a2d55557c87173cd090c9baa56979d20 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 7 Aug 2026 06:20:37 +0000
Subject: [PATCH 7/8] Fixups
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 1 -
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 5 +
.../RISCV/rvv/intrinsic-vector-match.ll | 362 ++++++------------
3 files changed, 128 insertions(+), 240 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index c1ab43d35ae17..d85bdcadab68f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -13890,7 +13890,6 @@ SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
}
-
SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ea4803e59ebc1..8236f1fa18f7b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2022,6 +2022,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
if (Subtarget.hasVInstructions())
setTargetDAGCombine({ISD::FCOPYSIGN,
+ ISD::VECTOR_MATCH,
ISD::MGATHER,
ISD::MSCATTER,
ISD::VP_GATHER,
@@ -23046,6 +23047,10 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
SDValue Fneg = DAG.getNode(ISD::FNEG, DL, VT, Splat);
return DAG.getNode(ISD::FMA, DL, VT, Fneg, N1, N->getOperand(2));
}
+ case ISD::VECTOR_MATCH:
+ if (DCI.isBeforeLegalize())
+ return expandVectorMatch(N, DAG);
+ return SDValue();
case ISD::SETCC:
return performSETCCCombine(N, DCI, Subtarget);
case ISD::SIGN_EXTEND_INREG:
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index 78fd483a786b8..6febcd22e3288 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -383,30 +383,22 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-LABEL: match_v8i8_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vmv.x.s a0, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 5
-; CHECK-NEXT: vmv.x.s a1, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 6
-; CHECK-NEXT: vmv.x.s a2, v10
-; CHECK-NEXT: vslidedown.vi v10, v9, 7
-; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 8
-; CHECK-NEXT: vmv.x.s a4, v10
+; CHECK-NEXT: vmv.x.s a0, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 9
-; CHECK-NEXT: vmv.x.s a5, v10
+; CHECK-NEXT: vmv.x.s a1, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 10
-; CHECK-NEXT: vmv.x.s a6, v10
+; CHECK-NEXT: vmv.x.s a2, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 11
-; CHECK-NEXT: vmv.x.s a7, v10
+; CHECK-NEXT: vmv.x.s a3, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 12
-; CHECK-NEXT: vmv.x.s t0, v10
+; CHECK-NEXT: vmv.x.s a4, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 13
-; CHECK-NEXT: vmv.x.s t1, v10
+; CHECK-NEXT: vmv.x.s a5, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 14
-; CHECK-NEXT: vmv.x.s t2, v10
+; CHECK-NEXT: vmv.x.s a6, v10
; CHECK-NEXT: vslidedown.vi v10, v9, 15
-; CHECK-NEXT: vmv.x.s t3, v10
+; CHECK-NEXT: vmv.x.s a7, v10
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vrgather.vi v10, v9, 1
; CHECK-NEXT: vrgather.vi v11, v9, 0
@@ -416,6 +408,18 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v10, v11, v10
; CHECK-NEXT: vmseq.vv v11, v8, v12
; CHECK-NEXT: vrgather.vi v12, v9, 3
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vmseq.vv v11, v8, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 4
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vmseq.vv v11, v8, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 5
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vmseq.vv v11, v8, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 6
+; CHECK-NEXT: vmor.mm v10, v10, v11
+; CHECK-NEXT: vmseq.vv v11, v8, v12
+; CHECK-NEXT: vrgather.vi v12, v9, 7
; CHECK-NEXT: vmor.mm v9, v10, v11
; CHECK-NEXT: vmseq.vv v10, v8, v12
; CHECK-NEXT: vmseq.vx v11, v8, a0
@@ -432,16 +436,8 @@ define <8 x i1> @match_v8i8_v16i8(<8 x i8> %op1, <16 x i8> %op2, <8 x i1> %mask)
; CHECK-NEXT: vmor.mm v9, v9, v11
; CHECK-NEXT: vmseq.vx v11, v8, a6
; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, a7
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, t0
-; CHECK-NEXT: vmor.mm v9, v9, v10
-; CHECK-NEXT: vmseq.vx v10, v8, t1
-; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v11, v8, t2
-; CHECK-NEXT: vmor.mm v9, v9, v10
; CHECK-NEXT: vmor.mm v9, v9, v11
-; CHECK-NEXT: vmseq.vx v8, v8, t3
+; CHECK-NEXT: vmseq.vx v8, v8, a7
; CHECK-NEXT: vmor.mm v8, v9, v8
; CHECK-NEXT: vmand.mm v0, v8, v0
; CHECK-NEXT: ret
@@ -716,76 +712,43 @@ define <vscale x 16 x i1> @match_nxv16i8_v32i8(<vscale x 16 x i8> %op1, <32 x i8
define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask) {
; RV32-LABEL: match_v16i8_v32i8:
; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -48
-; RV32-NEXT: .cfi_def_cfa_offset 48
-; RV32-NEXT: sw s0, 44(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s1, 40(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s2, 36(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s3, 32(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s4, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s5, 24(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s6, 20(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s7, 16(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s8, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset s0, -4
-; RV32-NEXT: .cfi_offset s1, -8
-; RV32-NEXT: .cfi_offset s2, -12
-; RV32-NEXT: .cfi_offset s3, -16
-; RV32-NEXT: .cfi_offset s4, -20
-; RV32-NEXT: .cfi_offset s5, -24
-; RV32-NEXT: .cfi_offset s6, -28
-; RV32-NEXT: .cfi_offset s7, -32
-; RV32-NEXT: .cfi_offset s8, -36
-; RV32-NEXT: vsetivli zero, 1, e8, m1, ta, ma
-; RV32-NEXT: vslidedown.vi v9, v10, 8
-; RV32-NEXT: vmv.x.s a0, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 9
-; RV32-NEXT: vmv.x.s a1, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 10
-; RV32-NEXT: vmv.x.s a2, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 11
-; RV32-NEXT: vmv.x.s a3, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 12
-; RV32-NEXT: vmv.x.s a4, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 13
-; RV32-NEXT: vmv.x.s a5, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 14
-; RV32-NEXT: vmv.x.s a6, v9
-; RV32-NEXT: vslidedown.vi v9, v10, 15
-; RV32-NEXT: vmv.x.s a7, v9
; RV32-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV32-NEXT: vslidedown.vi v12, v10, 16
-; RV32-NEXT: vmv.x.s t0, v12
+; RV32-NEXT: vmv.x.s a0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 17
-; RV32-NEXT: vmv.x.s t1, v12
+; RV32-NEXT: vmv.x.s a1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 18
-; RV32-NEXT: vmv.x.s t2, v12
+; RV32-NEXT: vmv.x.s a2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 19
-; RV32-NEXT: vmv.x.s t3, v12
+; RV32-NEXT: vmv.x.s a3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 20
-; RV32-NEXT: vmv.x.s t4, v12
+; RV32-NEXT: vmv.x.s a4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 21
-; RV32-NEXT: vmv.x.s t5, v12
+; RV32-NEXT: vmv.x.s a5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 22
-; RV32-NEXT: vmv.x.s t6, v12
+; RV32-NEXT: vmv.x.s a6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 23
-; RV32-NEXT: vmv.x.s s0, v12
+; RV32-NEXT: vmv.x.s a7, v12
; RV32-NEXT: vslidedown.vi v12, v10, 24
-; RV32-NEXT: vmv.x.s s1, v12
+; RV32-NEXT: vmv.x.s t0, v12
; RV32-NEXT: vslidedown.vi v12, v10, 25
-; RV32-NEXT: vmv.x.s s2, v12
+; RV32-NEXT: vmv.x.s t1, v12
; RV32-NEXT: vslidedown.vi v12, v10, 26
-; RV32-NEXT: vmv.x.s s3, v12
+; RV32-NEXT: vmv.x.s t2, v12
; RV32-NEXT: vslidedown.vi v12, v10, 27
-; RV32-NEXT: vmv.x.s s4, v12
+; RV32-NEXT: vmv.x.s t3, v12
; RV32-NEXT: vslidedown.vi v12, v10, 28
-; RV32-NEXT: vmv.x.s s5, v12
+; RV32-NEXT: vmv.x.s t4, v12
; RV32-NEXT: vslidedown.vi v12, v10, 29
-; RV32-NEXT: vmv.x.s s6, v12
+; RV32-NEXT: vmv.x.s t5, v12
; RV32-NEXT: vslidedown.vi v12, v10, 30
-; RV32-NEXT: vmv.x.s s7, v12
+; RV32-NEXT: vmv.x.s t6, v12
; RV32-NEXT: vslidedown.vi v12, v10, 31
-; RV32-NEXT: vmv.x.s s8, v12
+; RV32-NEXT: vmv.x.s s0, v12
; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vrgather.vi v9, v10, 1
; RV32-NEXT: vmseq.vv v9, v8, v9
@@ -807,6 +770,30 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmseq.vv v12, v8, v13
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vrgather.vi v11, v10, 7
+; RV32-NEXT: vrgather.vi v13, v10, 8
+; RV32-NEXT: vmor.mm v9, v9, v12
+; RV32-NEXT: vmseq.vv v11, v8, v11
+; RV32-NEXT: vmseq.vv v12, v8, v13
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vrgather.vi v11, v10, 9
+; RV32-NEXT: vrgather.vi v13, v10, 10
+; RV32-NEXT: vmor.mm v9, v9, v12
+; RV32-NEXT: vmseq.vv v11, v8, v11
+; RV32-NEXT: vmseq.vv v12, v8, v13
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vrgather.vi v11, v10, 11
+; RV32-NEXT: vrgather.vi v13, v10, 12
+; RV32-NEXT: vmor.mm v9, v9, v12
+; RV32-NEXT: vmseq.vv v11, v8, v11
+; RV32-NEXT: vmseq.vv v12, v8, v13
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vrgather.vi v11, v10, 13
+; RV32-NEXT: vrgather.vi v13, v10, 14
+; RV32-NEXT: vmor.mm v9, v9, v12
+; RV32-NEXT: vmseq.vv v11, v8, v11
+; RV32-NEXT: vmseq.vv v12, v8, v13
+; RV32-NEXT: vmor.mm v9, v9, v11
+; RV32-NEXT: vrgather.vi v11, v10, 15
; RV32-NEXT: vmor.mm v9, v9, v12
; RV32-NEXT: vmseq.vv v10, v8, v11
; RV32-NEXT: vmseq.vx v11, v8, a0
@@ -839,120 +826,55 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV32-NEXT: vmor.mm v9, v9, v11
; RV32-NEXT: vmseq.vx v11, v8, t6
; RV32-NEXT: vmor.mm v9, v9, v10
-; RV32-NEXT: vmseq.vx v10, v8, s0
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v11, v8, s1
-; RV32-NEXT: vmor.mm v9, v9, v10
-; RV32-NEXT: vmseq.vx v10, v8, s2
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v11, v8, s3
-; RV32-NEXT: vmor.mm v9, v9, v10
-; RV32-NEXT: vmseq.vx v10, v8, s4
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v11, v8, s5
-; RV32-NEXT: vmor.mm v9, v9, v10
-; RV32-NEXT: vmseq.vx v10, v8, s6
-; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v11, v8, s7
-; RV32-NEXT: vmor.mm v9, v9, v10
; RV32-NEXT: vmor.mm v9, v9, v11
-; RV32-NEXT: vmseq.vx v8, v8, s8
+; RV32-NEXT: vmseq.vx v8, v8, s0
; RV32-NEXT: vmor.mm v8, v9, v8
; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: lw s0, 44(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s1, 40(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s2, 36(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s3, 32(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s4, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s5, 24(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s6, 20(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s7, 16(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s8, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
; RV32-NEXT: .cfi_restore s0
-; RV32-NEXT: .cfi_restore s1
-; RV32-NEXT: .cfi_restore s2
-; RV32-NEXT: .cfi_restore s3
-; RV32-NEXT: .cfi_restore s4
-; RV32-NEXT: .cfi_restore s5
-; RV32-NEXT: .cfi_restore s6
-; RV32-NEXT: .cfi_restore s7
-; RV32-NEXT: .cfi_restore s8
-; RV32-NEXT: addi sp, sp, 48
+; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
;
; RV64-LABEL: match_v16i8_v32i8:
; RV64: # %bb.0:
-; RV64-NEXT: addi sp, sp, -80
-; RV64-NEXT: .cfi_def_cfa_offset 80
-; RV64-NEXT: sd s0, 72(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s1, 64(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s2, 56(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s3, 48(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s4, 40(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s5, 32(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s6, 24(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s7, 16(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s8, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: .cfi_def_cfa_offset 16
+; RV64-NEXT: sd s0, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset s0, -8
-; RV64-NEXT: .cfi_offset s1, -16
-; RV64-NEXT: .cfi_offset s2, -24
-; RV64-NEXT: .cfi_offset s3, -32
-; RV64-NEXT: .cfi_offset s4, -40
-; RV64-NEXT: .cfi_offset s5, -48
-; RV64-NEXT: .cfi_offset s6, -56
-; RV64-NEXT: .cfi_offset s7, -64
-; RV64-NEXT: .cfi_offset s8, -72
-; RV64-NEXT: vsetivli zero, 1, e8, m1, ta, ma
-; RV64-NEXT: vslidedown.vi v9, v10, 8
-; RV64-NEXT: vmv.x.s a0, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 9
-; RV64-NEXT: vmv.x.s a1, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 10
-; RV64-NEXT: vmv.x.s a2, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 11
-; RV64-NEXT: vmv.x.s a3, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 12
-; RV64-NEXT: vmv.x.s a4, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 13
-; RV64-NEXT: vmv.x.s a5, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 14
-; RV64-NEXT: vmv.x.s a6, v9
-; RV64-NEXT: vslidedown.vi v9, v10, 15
-; RV64-NEXT: vmv.x.s a7, v9
; RV64-NEXT: vsetivli zero, 1, e8, m2, ta, ma
; RV64-NEXT: vslidedown.vi v12, v10, 16
-; RV64-NEXT: vmv.x.s t0, v12
+; RV64-NEXT: vmv.x.s a0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 17
-; RV64-NEXT: vmv.x.s t1, v12
+; RV64-NEXT: vmv.x.s a1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 18
-; RV64-NEXT: vmv.x.s t2, v12
+; RV64-NEXT: vmv.x.s a2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 19
-; RV64-NEXT: vmv.x.s t3, v12
+; RV64-NEXT: vmv.x.s a3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 20
-; RV64-NEXT: vmv.x.s t4, v12
+; RV64-NEXT: vmv.x.s a4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 21
-; RV64-NEXT: vmv.x.s t5, v12
+; RV64-NEXT: vmv.x.s a5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 22
-; RV64-NEXT: vmv.x.s t6, v12
+; RV64-NEXT: vmv.x.s a6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 23
-; RV64-NEXT: vmv.x.s s0, v12
+; RV64-NEXT: vmv.x.s a7, v12
; RV64-NEXT: vslidedown.vi v12, v10, 24
-; RV64-NEXT: vmv.x.s s1, v12
+; RV64-NEXT: vmv.x.s t0, v12
; RV64-NEXT: vslidedown.vi v12, v10, 25
-; RV64-NEXT: vmv.x.s s2, v12
+; RV64-NEXT: vmv.x.s t1, v12
; RV64-NEXT: vslidedown.vi v12, v10, 26
-; RV64-NEXT: vmv.x.s s3, v12
+; RV64-NEXT: vmv.x.s t2, v12
; RV64-NEXT: vslidedown.vi v12, v10, 27
-; RV64-NEXT: vmv.x.s s4, v12
+; RV64-NEXT: vmv.x.s t3, v12
; RV64-NEXT: vslidedown.vi v12, v10, 28
-; RV64-NEXT: vmv.x.s s5, v12
+; RV64-NEXT: vmv.x.s t4, v12
; RV64-NEXT: vslidedown.vi v12, v10, 29
-; RV64-NEXT: vmv.x.s s6, v12
+; RV64-NEXT: vmv.x.s t5, v12
; RV64-NEXT: vslidedown.vi v12, v10, 30
-; RV64-NEXT: vmv.x.s s7, v12
+; RV64-NEXT: vmv.x.s t6, v12
; RV64-NEXT: vslidedown.vi v12, v10, 31
-; RV64-NEXT: vmv.x.s s8, v12
+; RV64-NEXT: vmv.x.s s0, v12
; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vrgather.vi v9, v10, 1
; RV64-NEXT: vmseq.vv v9, v8, v9
@@ -974,6 +896,30 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmseq.vv v12, v8, v13
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vrgather.vi v11, v10, 7
+; RV64-NEXT: vrgather.vi v13, v10, 8
+; RV64-NEXT: vmor.mm v9, v9, v12
+; RV64-NEXT: vmseq.vv v11, v8, v11
+; RV64-NEXT: vmseq.vv v12, v8, v13
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vrgather.vi v11, v10, 9
+; RV64-NEXT: vrgather.vi v13, v10, 10
+; RV64-NEXT: vmor.mm v9, v9, v12
+; RV64-NEXT: vmseq.vv v11, v8, v11
+; RV64-NEXT: vmseq.vv v12, v8, v13
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vrgather.vi v11, v10, 11
+; RV64-NEXT: vrgather.vi v13, v10, 12
+; RV64-NEXT: vmor.mm v9, v9, v12
+; RV64-NEXT: vmseq.vv v11, v8, v11
+; RV64-NEXT: vmseq.vv v12, v8, v13
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vrgather.vi v11, v10, 13
+; RV64-NEXT: vrgather.vi v13, v10, 14
+; RV64-NEXT: vmor.mm v9, v9, v12
+; RV64-NEXT: vmseq.vv v11, v8, v11
+; RV64-NEXT: vmseq.vv v12, v8, v13
+; RV64-NEXT: vmor.mm v9, v9, v11
+; RV64-NEXT: vrgather.vi v11, v10, 15
; RV64-NEXT: vmor.mm v9, v9, v12
; RV64-NEXT: vmseq.vv v10, v8, v11
; RV64-NEXT: vmseq.vx v11, v8, a0
@@ -1006,45 +952,13 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
; RV64-NEXT: vmor.mm v9, v9, v11
; RV64-NEXT: vmseq.vx v11, v8, t6
; RV64-NEXT: vmor.mm v9, v9, v10
-; RV64-NEXT: vmseq.vx v10, v8, s0
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v11, v8, s1
-; RV64-NEXT: vmor.mm v9, v9, v10
-; RV64-NEXT: vmseq.vx v10, v8, s2
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v11, v8, s3
-; RV64-NEXT: vmor.mm v9, v9, v10
-; RV64-NEXT: vmseq.vx v10, v8, s4
; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v11, v8, s5
-; RV64-NEXT: vmor.mm v9, v9, v10
-; RV64-NEXT: vmseq.vx v10, v8, s6
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v11, v8, s7
-; RV64-NEXT: vmor.mm v9, v9, v10
-; RV64-NEXT: vmor.mm v9, v9, v11
-; RV64-NEXT: vmseq.vx v8, v8, s8
+; RV64-NEXT: vmseq.vx v8, v8, s0
; RV64-NEXT: vmor.mm v8, v9, v8
; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ld s0, 72(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s1, 64(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s2, 56(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s3, 48(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s4, 40(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s5, 32(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s6, 24(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s7, 16(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s8, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 8(sp) # 8-byte Folded Reload
; RV64-NEXT: .cfi_restore s0
-; RV64-NEXT: .cfi_restore s1
-; RV64-NEXT: .cfi_restore s2
-; RV64-NEXT: .cfi_restore s3
-; RV64-NEXT: .cfi_restore s4
-; RV64-NEXT: .cfi_restore s5
-; RV64-NEXT: .cfi_restore s6
-; RV64-NEXT: .cfi_restore s7
-; RV64-NEXT: .cfi_restore s8
-; RV64-NEXT: addi sp, sp, 80
+; RV64-NEXT: addi sp, sp, 16
; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %mask)
@@ -1073,46 +987,16 @@ define <vscale x 4 x i1> @match_nxv4xi32_v4i32(<vscale x 4 x i32> %op1, <4 x i32
}
define <vscale x 2 x i1> @match_nxv2xi64_v2i64(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask) {
-; RV32-LABEL: match_nxv2xi64_v2i64:
-; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -16
-; RV32-NEXT: .cfi_def_cfa_offset 16
-; RV32-NEXT: li a0, 32
-; RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT: vmv.x.s a1, v10
-; RV32-NEXT: vsrl.vx v11, v10, a0
-; RV32-NEXT: vmv.x.s a2, v11
-; RV32-NEXT: vslidedown.vi v10, v10, 1
-; RV32-NEXT: vmv.x.s a3, v10
-; RV32-NEXT: vsrl.vx v10, v10, a0
-; RV32-NEXT: vmv.x.s a0, v10
-; RV32-NEXT: sw a1, 8(sp)
-; RV32-NEXT: sw a2, 12(sp)
-; RV32-NEXT: sw a3, 0(sp)
-; RV32-NEXT: sw a0, 4(sp)
-; RV32-NEXT: addi a0, sp, 8
-; RV32-NEXT: vsetvli a1, zero, e64, m2, ta, ma
-; RV32-NEXT: vlse64.v v10, (a0), zero
-; RV32-NEXT: mv a0, sp
-; RV32-NEXT: vlse64.v v12, (a0), zero
-; RV32-NEXT: vmseq.vv v14, v8, v10
-; RV32-NEXT: vmseq.vv v10, v8, v12
-; RV32-NEXT: vmor.mm v8, v14, v10
-; RV32-NEXT: vmand.mm v0, v8, v0
-; RV32-NEXT: addi sp, sp, 16
-; RV32-NEXT: .cfi_def_cfa_offset 0
-; RV32-NEXT: ret
-;
-; RV64-LABEL: match_nxv2xi64_v2i64:
-; RV64: # %bb.0:
-; RV64-NEXT: vsetvli a0, zero, e64, m2, ta, ma
-; RV64-NEXT: vrgather.vi v12, v10, 1
-; RV64-NEXT: vmseq.vv v14, v8, v12
-; RV64-NEXT: vrgather.vi v12, v10, 0
-; RV64-NEXT: vmseq.vv v10, v8, v12
-; RV64-NEXT: vmor.mm v8, v10, v14
-; RV64-NEXT: vmand.mm v0, v8, v0
-; RV64-NEXT: ret
+; CHECK-LABEL: match_nxv2xi64_v2i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT: vrgather.vi v12, v10, 1
+; CHECK-NEXT: vmseq.vv v14, v8, v12
+; CHECK-NEXT: vrgather.vi v12, v10, 0
+; CHECK-NEXT: vmseq.vv v10, v8, v12
+; CHECK-NEXT: vmor.mm v8, v10, v14
+; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: ret
%r = tail call <vscale x 2 x i1> @llvm.experimental.vector.match(<vscale x 2 x i64> %op1, <2 x i64> %op2, <vscale x 2 x i1> %mask)
ret <vscale x 2 x i1> %r
}
>From 0366ae563be3a0f3dccefb9e26677b08f2b14257 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 7 Aug 2026 07:14:36 +0000
Subject: [PATCH 8/8] Fixups
---
.../SelectionDAG/LegalizeIntegerTypes.cpp | 4 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 43 +++++++++----------
.../AArch64/intrinsic-vector-match-sve2.ll | 43 +++++++++++++++++++
3 files changed, 67 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 59b2098ea9bac..b4c2c8376dbe6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -6624,7 +6624,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_MATCH(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
- return DAG.getNode(ISD::VECTOR_MATCH, SDLoc(N), NVT, N->ops());
+ SmallVector<SDValue, 3> NewOps(N->ops());
+ NewOps[2] = PromoteTargetBoolean(N->getOperand(2), NVT);
+ return DAG.getNode(ISD::VECTOR_MATCH, SDLoc(N), NVT, NewOps, N->getFlags());
}
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index d7809e24beb23..8eb16377a4ad1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5251,29 +5251,25 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
SDLoc DL(N);
- if (OpNo != 1) {
- auto SplitOperand = [&](SDValue Op, SDValue &Lo, SDValue &Hi) {
- if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSplitVector)
- GetSplitVector(Op, Lo, Hi);
- else
- std::tie(Lo, Hi) = DAG.SplitVector(Op, DL);
- };
-
+ if (OpNo == 0) {
+ EVT LoResVT, HiResVT;
+ std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(N->getValueType(0));
SDValue SourceLo, SourceHi;
- SplitOperand(N->getOperand(0), SourceLo, SourceHi);
+ std::tie(SourceLo, SourceHi) = DAG.SplitVectorOperand(N, 0);
SDValue MaskLo, MaskHi;
- SplitOperand(N->getOperand(2), MaskLo, MaskHi);
-
- SDValue MatchLo =
- DAG.getNode(ISD::VECTOR_MATCH, DL, MaskLo.getValueType(), SourceLo,
- N->getOperand(1), MaskLo, N->getFlags());
- SDValue MatchHi =
- DAG.getNode(ISD::VECTOR_MATCH, DL, MaskHi.getValueType(), SourceHi,
- N->getOperand(1), MaskHi, N->getFlags());
+ std::tie(MaskLo, MaskHi) = DAG.SplitVectorOperand(N, 2);
+
+ SDValue MatchLo = DAG.getNode(ISD::VECTOR_MATCH, DL, LoResVT, SourceLo,
+ N->getOperand(1), MaskLo, N->getFlags());
+ SDValue MatchHi = DAG.getNode(ISD::VECTOR_MATCH, DL, HiResVT, SourceHi,
+ N->getOperand(1), MaskHi, N->getFlags());
return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0), MatchLo,
MatchHi);
}
+ // Note: The Mask (OpNo == 2) should be widened with the result.
+ assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
+
SDValue NeedleLo, NeedleHi;
GetSplitVector(N->getOperand(1), NeedleLo, NeedleHi);
@@ -8951,14 +8947,14 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
}
SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
- if (OpNo != 1) {
+ if (OpNo == 0) {
SDLoc DL(N);
EVT ResVT = N->getValueType(0);
- EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
EVT SourceVT = N->getOperand(0).getValueType();
- EVT WideSourceVT =
- EVT::getVectorVT(*DAG.getContext(), SourceVT.getVectorElementType(),
- WidenVT.getVectorElementCount());
+ EVT WideSourceVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
+ EVT WidenVT =
+ EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
+ WideSourceVT.getVectorElementCount());
SDValue WideSource = DAG.getInsertSubvector(DL, DAG.getUNDEF(WideSourceVT),
N->getOperand(0), 0);
@@ -8969,6 +8965,9 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
return DAG.getExtractSubvector(DL, ResVT, WideMatch, 0);
}
+ // Note: The Mask (OpNo == 2) should be widened with the result.
+ assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
+
SDLoc DL(N);
SDValue Needle = N->getOperand(1);
EVT NeedleVT = Needle.getValueType();
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 82912df0c8fee..0e0fa7689de44 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -12,6 +12,26 @@ define <vscale x 16 x i1> @match_nxv16i8_v1i8(<vscale x 16 x i8> %op1, <1 x i8>
ret <vscale x 16 x i1> %r
}
+define <vscale x 1 x i1> @match_nxv1i8_v8i8(<vscale x 1 x i8> %op1, <8 x i8> %op2, <vscale x 1 x i1> %mask) #0 {
+; CHECK-LABEL: match_nxv1i8_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT: mov z1.d, d1
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT: match p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: punpklo p0.h, p1.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+ %r = tail call <vscale x 1 x i1> @llvm.experimental.vector.match(<vscale x 1 x i8> %op1, <8 x i8> %op2, <vscale x 1 x i1> %mask)
+ ret <vscale x 1 x i1> %r
+}
+
define <1 x i1> @match_v1i8_v1i8(<1 x i8> %op1, <1 x i8> %op2, <1 x i1> %mask) #0 {
; CHECK-LABEL: match_v1i8_v1i8:
; CHECK: // %bb.0:
@@ -273,6 +293,29 @@ define <16 x i1> @match_v16i8_v32i8(<16 x i8> %op1, <32 x i8> %op2, <16 x i1> %m
ret <16 x i1> %r
}
+define <16 x i1> @match_v16i16_v8i16(<16 x i16> %op1, <8 x i16> %op2, <16 x i1> %mask) #0 {
+; CHECK-LABEL: match_v16i16_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll2 v4.8h, v3.16b, #0
+; CHECK-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: shl v4.8h, v4.8h, #15
+; CHECK-NEXT: shl v3.8h, v3.8h, #15
+; CHECK-NEXT: cmpne p1.h, p0/z, z4.h, #0
+; CHECK-NEXT: cmpne p2.h, p0/z, z3.h, #0
+; CHECK-NEXT: match p0.h, p1/z, z1.h, z2.h
+; CHECK-NEXT: match p1.h, p2/z, z0.h, z2.h
+; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: mov z1.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+ %r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i16> %op1, <8 x i16> %op2, <16 x i1> %mask)
+ ret <16 x i1> %r
+}
+
define <vscale x 32 x i1> @match_nxv32i8_v16i8(<vscale x 32 x i8> %op1, <16 x i8> %op2, <vscale x 32 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv32i8_v16i8:
; CHECK: // %bb.0:
More information about the llvm-commits
mailing list