[llvm] [AArch64] Add DAG combine to widen v3i8/v4i8 VECTOR_MATCH needles (PR #218972)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 07:15:05 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/218972
>From 0b56b5c8dce22278990757c7907ac7bd8d593248 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 26 Aug 2026 15:56:45 +0000
Subject: [PATCH 1/3] Precommit tests
---
.../AArch64/intrinsic-vector-match-sve2.ll | 36 +++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 8771fcaaaee63..52c66dbacd910 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -60,6 +60,24 @@ define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8>
ret <vscale x 16 x i1> %r
}
+define <vscale x 16 x i1> @match_nxv16i8_v3i8(<vscale x 16 x i8> %op1, <3 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
+; CHECK-LABEL: match_nxv16i8_v3i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.b, w1
+; CHECK-NEXT: mov z2.b, w0
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: mov z3.b, w2
+; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b
+; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z2.b
+; CHECK-NEXT: cmpeq p1.b, p1/z, z0.b, z3.b
+; CHECK-NEXT: mov p2.b, p3/m, p3.b
+; CHECK-NEXT: mov p1.b, p2/m, p2.b
+; CHECK-NEXT: orr p0.b, p0/z, p1.b, p3.b
+; CHECK-NEXT: ret
+ %r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <3 x i8> %op2, <vscale x 16 x i1> %mask)
+ ret <vscale x 16 x i1> %r
+}
+
define <vscale x 16 x i1> @match_nxv16i8_v4i8(<vscale x 16 x i8> %op1, <4 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v4i8:
; CHECK: // %bb.0:
@@ -142,6 +160,24 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
ret <16 x i1> %r
}
+define <16 x i1> @match_v16i8_v3i8(<16 x i8> %op1, <3 x i8> %op2, <16 x i1> %mask) #0 {
+; CHECK-LABEL: match_v16i8_v3i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dup v2.16b, w1
+; CHECK-NEXT: dup v3.16b, w0
+; CHECK-NEXT: dup v4.16b, w2
+; CHECK-NEXT: cmeq v2.16b, v0.16b, v2.16b
+; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
+; CHECK-NEXT: cmeq v0.16b, v0.16b, v4.16b
+; CHECK-NEXT: orr v2.16b, v3.16b, v2.16b
+; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <3 x i8> %op2, <16 x i1> %mask)
+ ret <16 x i1> %r
+}
+
define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask) #0 {
; CHECK-LABEL: match_v16i8_v4i8:
; CHECK: // %bb.0:
>From 25fdd471eaa866ff148dbd3f063fe237335145da Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 26 Aug 2026 16:00:46 +0000
Subject: [PATCH 2/3] [AArch64] Add DAG combine to widen v3i8/v4i8 VECTOR_MATCH
needles
The default legalization behaviour for v3i8 and v4i8 types is to promote
them to v4i16. This currently results in them being expanded for AArch64.
It's generally preferable to widen the needle instead (especially if the
needle is a constant vector, which becomes an immediate splat). For SVE,
this can fold multiple vector compares into a dup + match.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 44 +++++++++-
.../AArch64/intrinsic-vector-match-sve2.ll | 88 +++++++------------
2 files changed, 73 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4606040d4ce38..401be4c8bd842 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1656,6 +1656,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
+
+ setTargetDAGCombine(ISD::VECTOR_MATCH);
}
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
@@ -30855,7 +30857,11 @@ static SDValue performBSPExpandForSVE(SDNode *N, SelectionDAG &DAG,
static SDValue performDupLane128Combine(SDNode *N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
- SDValue Insert = N->getOperand(0);
+ SDValue Op = N->getOperand(0);
+ if (Op.getOpcode() == ISD::SPLAT_VECTOR && Op.getValueType() == VT)
+ return Op;
+
+ SDValue Insert = Op;
if (Insert.getOpcode() != ISD::INSERT_SUBVECTOR)
return SDValue();
@@ -31451,6 +31457,40 @@ static SDValue performPredicateLoadCombine(SDNode *N,
return LoadPred;
}
+static SDValue performVectorMatchCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG) {
+ // Widen v3i8/v4i8 match needles to v8i8. For needles >= 2 elements it's
+ // generally better to widen the needle and lower to a `match` (rather than
+ // expanding). This is handled by type legalization for all other types (> 2
+ // elements) but v4i8 is promoted rather than widened, hence this combine.
+ SDValue Needle = N->getOperand(1);
+ EVT NeedleVT = Needle.getValueType();
+
+ if (!DCI.isBeforeLegalize() ||
+ (NeedleVT != MVT::v3i8 && NeedleVT != MVT::v4i8))
+ return SDValue();
+
+ SDLoc DL(N);
+ if (NeedleVT == MVT::v3i8) {
+ // Pad a v3i8 needle to v4i8.
+ SDValue Pad = DAG.getExtractVectorElt(DL, MVT::i8, Needle, 0);
+ Needle = DAG.getInsertSubvector(DL, DAG.getPOISON(MVT::v4i8), Needle, 0);
+ Needle = DAG.getInsertVectorElt(DL, Needle, Pad, 3);
+ }
+
+ // Splat the needle to a full scalable vector (in such a way that the bitcasts
+ // and extracts should fold away).
+ Needle = DAG.getBitcast(MVT::v1i32, Needle);
+ Needle = DAG.getExtractVectorElt(DL, MVT::i32, Needle, 0);
+ Needle = DAG.getSplatVector(MVT::nxv4i32, DL, Needle);
+ Needle = DAG.getBitcast(MVT::nxv16i8, Needle);
+ Needle = DAG.getExtractSubvector(DL, MVT::v16i8, Needle, 0);
+
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0),
+ N->getOperand(0), Needle, N->getOperand(2));
+}
+
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
@@ -31814,6 +31854,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performCTPOPCombine(N, DCI, DAG);
case ISD::BITCAST:
return performPredicateLoadCombine(N, DCI, DAG);
+ case ISD::VECTOR_MATCH:
+ return performVectorMatchCombine(N, DCI, DAG);
}
return SDValue();
}
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 52c66dbacd910..36f2273148ae9 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -63,16 +63,12 @@ define <vscale x 16 x i1> @match_nxv16i8_v2i8(<vscale x 16 x i8> %op1, <2 x i8>
define <vscale x 16 x i1> @match_nxv16i8_v3i8(<vscale x 16 x i8> %op1, <3 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v3i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.b, w1
-; CHECK-NEXT: mov z2.b, w0
-; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: mov z3.b, w2
-; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z2.b
-; CHECK-NEXT: cmpeq p1.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov p2.b, p3/m, p3.b
-; CHECK-NEXT: mov p1.b, p2/m, p2.b
-; CHECK-NEXT: orr p0.b, p0/z, p1.b, p3.b
+; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: mov v1.b[1], w1
+; CHECK-NEXT: mov v1.b[2], w2
+; CHECK-NEXT: mov v1.b[3], w0
+; CHECK-NEXT: mov z1.s, s1
+; CHECK-NEXT: match p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT: ret
%r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <3 x i8> %op2, <vscale x 16 x i1> %mask)
ret <vscale x 16 x i1> %r
@@ -81,31 +77,9 @@ define <vscale x 16 x i1> @match_nxv16i8_v3i8(<vscale x 16 x i8> %op1, <3 x i8>
define <vscale x 16 x i1> @match_nxv16i8_v4i8(<vscale x 16 x i8> %op1, <4 x i8> %op2, <vscale x 16 x i1> %mask) #0 {
; CHECK-LABEL: match_nxv16i8_v4i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Spill
-; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: umov w8, v1.h[1]
-; CHECK-NEXT: umov w9, v1.h[0]
-; CHECK-NEXT: umov w10, v1.h[2]
-; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: mov z2.b, w8
-; CHECK-NEXT: mov z3.b, w9
-; CHECK-NEXT: umov w8, v1.h[3]
-; CHECK-NEXT: mov z1.b, w10
-; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z2.b
-; CHECK-NEXT: cmpeq p3.b, p1/z, z0.b, z3.b
-; CHECK-NEXT: mov z2.b, w8
-; CHECK-NEXT: cmpeq p4.b, p1/z, z0.b, z1.b
-; CHECK-NEXT: cmpeq p1.b, p1/z, z0.b, z2.b
-; CHECK-NEXT: mov p2.b, p3/m, p3.b
-; CHECK-NEXT: sel p2.b, p2, p2.b, p4.b
-; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Reload
-; CHECK-NEXT: orr p0.b, p0/z, p2.b, p1.b
-; CHECK-NEXT: addvl sp, sp, #1
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-NEXT: mov z1.s, s1
+; CHECK-NEXT: match p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT: ret
%r = tail call <vscale x 16 x i1> @llvm.experimental.vector.match(<vscale x 16 x i8> %op1, <4 x i8> %op2, <vscale x 16 x i1> %mask)
ret <vscale x 16 x i1> %r
@@ -163,16 +137,18 @@ define <16 x i1> @match_v16i8_v2i8(<16 x i8> %op1, <2 x i8> %op2, <16 x i1> %mas
define <16 x i1> @match_v16i8_v3i8(<16 x i8> %op1, <3 x i8> %op2, <16 x i1> %mask) #0 {
; CHECK-LABEL: match_v16i8_v3i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: dup v2.16b, w1
-; CHECK-NEXT: dup v3.16b, w0
-; CHECK-NEXT: dup v4.16b, w2
-; CHECK-NEXT: cmeq v2.16b, v0.16b, v2.16b
-; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
-; CHECK-NEXT: cmeq v0.16b, v0.16b, v4.16b
-; CHECK-NEXT: orr v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: fmov s2, w0
+; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: mov v2.b[1], w1
+; CHECK-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-NEXT: mov v2.b[2], w2
+; CHECK-NEXT: mov v2.b[3], w0
+; CHECK-NEXT: mov z1.s, s2
+; CHECK-NEXT: match p0.b, p1/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <3 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
@@ -181,19 +157,15 @@ define <16 x i1> @match_v16i8_v3i8(<16 x i8> %op1, <3 x i8> %op2, <16 x i1> %mas
define <16 x i1> @match_v16i8_v4i8(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask) #0 {
; CHECK-LABEL: match_v16i8_v4i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: dup v3.16b, v1.b[2]
-; CHECK-NEXT: dup v4.16b, v1.b[0]
-; CHECK-NEXT: dup v5.16b, v1.b[4]
-; CHECK-NEXT: dup v1.16b, v1.b[6]
-; CHECK-NEXT: cmeq v3.16b, v0.16b, v3.16b
-; CHECK-NEXT: cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT: cmeq v5.16b, v0.16b, v5.16b
-; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: orr v1.16b, v4.16b, v3.16b
-; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
-; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-NEXT: shl v2.16b, v2.16b, #7
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: cmpne p1.b, p0/z, z2.b, #0
+; CHECK-NEXT: mov z1.s, s1
+; CHECK-NEXT: match p0.b, p1/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <4 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
>From 38473537985656d78390cbcea44d917901e0233e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 27 Aug 2026 14:08:56 +0000
Subject: [PATCH 3/3] Fixups
---
.../Target/AArch64/AArch64ISelLowering.cpp | 66 ++++++++-----------
.../AArch64/intrinsic-vector-match-sve2.ll | 65 ++++++++++++++++++
2 files changed, 92 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 401be4c8bd842..81a6683d52de1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1654,10 +1654,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
- for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
+ for (MVT VT :
+ {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8, MVT::v3i8, MVT::v4i8})
setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
-
- setTargetDAGCombine(ISD::VECTOR_MATCH);
}
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
@@ -6515,6 +6514,31 @@ static SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
EVT Op2VT = Op2.getValueType();
EVT ResVT = Op.getValueType();
+ if ((Op2VT == MVT::v3i8 || Op2VT == MVT::v4i8)) {
+ SDValue Needle = Op2;
+ EVT NeedleVT = Op2VT;
+
+ if (NeedleVT == MVT::v3i8) {
+ // Pad a v3i8 needle to v4i8.
+ SDValue Pad = DAG.getExtractVectorElt(DL, MVT::i8, Needle, 0);
+ Needle = DAG.getInsertSubvector(DL, DAG.getPOISON(MVT::v4i8), Needle, 0);
+ Needle = DAG.getInsertVectorElt(DL, Needle, Pad, 3);
+ }
+
+ // Extend the v4i8 needle to v16i8.
+ Needle = DAG.getBitcast(MVT::v1i32, Needle);
+ Needle = DAG.getExtractVectorElt(DL, MVT::i32, Needle, 0);
+ Needle = DAG.getSplatVector(MVT::v4i32, DL, Needle);
+ Needle = DAG.getBitcast(MVT::v16i8, Needle);
+
+ return DAG.getNode(ISD::VECTOR_MATCH, DL, Op.getValueType(), Op1, Needle,
+ Mask);
+ }
+
+ if (!DAG.getTargetLoweringInfo().isTypeLegal(ResVT) ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(Op1VT))
+ return SDValue();
+
assert((Op1VT.getVectorElementType() == MVT::i8 ||
Op1VT.getVectorElementType() == MVT::i16) &&
"Expected 8-bit or 16-bit characters.");
@@ -31457,40 +31481,6 @@ static SDValue performPredicateLoadCombine(SDNode *N,
return LoadPred;
}
-static SDValue performVectorMatchCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI,
- SelectionDAG &DAG) {
- // Widen v3i8/v4i8 match needles to v8i8. For needles >= 2 elements it's
- // generally better to widen the needle and lower to a `match` (rather than
- // expanding). This is handled by type legalization for all other types (> 2
- // elements) but v4i8 is promoted rather than widened, hence this combine.
- SDValue Needle = N->getOperand(1);
- EVT NeedleVT = Needle.getValueType();
-
- if (!DCI.isBeforeLegalize() ||
- (NeedleVT != MVT::v3i8 && NeedleVT != MVT::v4i8))
- return SDValue();
-
- SDLoc DL(N);
- if (NeedleVT == MVT::v3i8) {
- // Pad a v3i8 needle to v4i8.
- SDValue Pad = DAG.getExtractVectorElt(DL, MVT::i8, Needle, 0);
- Needle = DAG.getInsertSubvector(DL, DAG.getPOISON(MVT::v4i8), Needle, 0);
- Needle = DAG.getInsertVectorElt(DL, Needle, Pad, 3);
- }
-
- // Splat the needle to a full scalable vector (in such a way that the bitcasts
- // and extracts should fold away).
- Needle = DAG.getBitcast(MVT::v1i32, Needle);
- Needle = DAG.getExtractVectorElt(DL, MVT::i32, Needle, 0);
- Needle = DAG.getSplatVector(MVT::nxv4i32, DL, Needle);
- Needle = DAG.getBitcast(MVT::nxv16i8, Needle);
- Needle = DAG.getExtractSubvector(DL, MVT::v16i8, Needle, 0);
-
- return DAG.getNode(ISD::VECTOR_MATCH, DL, N->getValueType(0),
- N->getOperand(0), Needle, N->getOperand(2));
-}
-
SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
@@ -31854,8 +31844,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performCTPOPCombine(N, DCI, DAG);
case ISD::BITCAST:
return performPredicateLoadCombine(N, DCI, DAG);
- case ISD::VECTOR_MATCH:
- return performVectorMatchCombine(N, DCI, DAG);
}
return SDValue();
}
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
index 36f2273148ae9..62b20ef9694ce 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-vector-match-sve2.ll
@@ -450,4 +450,69 @@ define <2 x i1> @match_v2xi64_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %ma
ret <2 x i1> %r
}
+; Ensure we don't try to custom lower nodes with v3i8 source operands.
+; TODO: It may be worth extending the source operand so this case can use match.
+define <3 x i1> @match_v3i8_v3i1(<3 x i8> %op1, <8 x i8> %op2, <3 x i1> %mask) #0 {
+; CHECK-LABEL: match_v3i8_v3i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[1]
+; CHECK-NEXT: umov w9, v0.b[0]
+; CHECK-NEXT: umov w10, v0.b[2]
+; CHECK-NEXT: umov w11, v0.b[3]
+; CHECK-NEXT: umov w12, v0.b[4]
+; CHECK-NEXT: umov w13, v0.b[5]
+; CHECK-NEXT: mov v1.h[1], w1
+; CHECK-NEXT: dup v2.4h, w8
+; CHECK-NEXT: umov w8, v0.b[6]
+; CHECK-NEXT: dup v3.4h, w9
+; CHECK-NEXT: dup v4.4h, w10
+; CHECK-NEXT: dup v5.4h, w11
+; CHECK-NEXT: dup v6.4h, w12
+; CHECK-NEXT: dup v7.4h, w13
+; CHECK-NEXT: mov v1.h[2], w2
+; CHECK-NEXT: dup v16.4h, w8
+; CHECK-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NEXT: bic v4.4h, #255, lsl #8
+; CHECK-NEXT: bic v5.4h, #255, lsl #8
+; CHECK-NEXT: bic v6.4h, #255, lsl #8
+; CHECK-NEXT: bic v7.4h, #255, lsl #8
+; CHECK-NEXT: umov w8, v0.b[7]
+; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: bic v16.4h, #255, lsl #8
+; CHECK-NEXT: cmeq v0.4h, v1.4h, v2.4h
+; CHECK-NEXT: cmeq v2.4h, v1.4h, v3.4h
+; CHECK-NEXT: cmeq v3.4h, v1.4h, v4.4h
+; CHECK-NEXT: cmeq v4.4h, v1.4h, v5.4h
+; CHECK-NEXT: cmeq v5.4h, v1.4h, v6.4h
+; CHECK-NEXT: cmeq v6.4h, v1.4h, v7.4h
+; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: orr v2.8b, v3.8b, v4.8b
+; CHECK-NEXT: orr v4.8b, v5.8b, v6.8b
+; CHECK-NEXT: cmeq v5.4h, v1.4h, v16.4h
+; CHECK-NEXT: dup v3.4h, w8
+; CHECK-NEXT: orr v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: orr v2.8b, v4.8b, v5.8b
+; CHECK-NEXT: fmov s4, w3
+; CHECK-NEXT: bic v3.4h, #255, lsl #8
+; CHECK-NEXT: mov v4.h[1], w4
+; CHECK-NEXT: orr v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: cmeq v1.4h, v1.4h, v3.4h
+; CHECK-NEXT: mov v4.h[2], w5
+; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NEXT: shl v1.4h, v4.4h, #15
+; CHECK-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: umov w0, v0.h[0]
+; CHECK-NEXT: umov w1, v0.h[1]
+; CHECK-NEXT: umov w2, v0.h[2]
+; CHECK-NEXT: ret
+ %r = tail call <3 x i1> @llvm.experimental.vector.match(<3 x i8> %op1, <8 x i8> %op2, <3 x i1> %mask)
+ ret <3 x i1> %r
+}
+
attributes #0 = { "target-features"="+sve2" }
More information about the llvm-commits
mailing list