[llvm-branch-commits] [llvm] [AArch64] Add patterns to combine SVE AND and XOR to BIC (PR #238442)
Vimal Patel via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 02:44:53 PST 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/238442
>From bdc35d4f52211f51f9fa8ff802331026552b90f2 Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Wed, 4 Nov 2026 13:55:44 +0000
Subject: [PATCH 1/4] [AArch64] Add patterns to combine SVE AND and XOR to BIC
Add ISel patterns to rewrite the following pattern to BIC:
```
return svand_u32_x(svptrue_b32(), a, sveor_u32_x(svptrue_b32(), a, b));
```
Cover cases where atleast one of AND and XOR node is of _u form or is a
normal LLVM AND/XOR node.
Also, combine sve.and.u and sve.eor(.z) when predicate for sve.and.u
matches that of sve.eor(.z) or the predicate of sve.eor(.z) is an
all-active predicate.
Cover all SVE integer widths and cover the following symmetric case:
```
return sveor_u32_x(svptrue_b32(), a, svand_u32_x(svptrue_b32(), a, b));
```
---
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 123 ++++++++++++++++--
.../CodeGen/AArch64/sve-intrinsics-and-xor.ll | 45 ++++---
.../CodeGen/AArch64/sve-intrinsics-xor-and.ll | 45 ++++---
4 files changed, 169 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a51e6bb18cdd8e..19214f4469c23c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25890,8 +25890,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_eor_u:
- return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return DAG.getNode(AArch64ISD::EOR_U, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_orr_u:
return DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 86898c3139c69c..7c51d5a2ffd19d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -252,6 +252,7 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [
// Predicated operations with the result of inactive lanes being unspecified.
def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>;
+def AArch64eor_u : SDNode<"AArch64ISD::EOR_U", SDT_AArch64Arith>;
def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>;
def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>;
def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>;
@@ -560,10 +561,17 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
-// Match AND_U with either data operand order, keeping the predicate unchanged.
-def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
- [(AArch64and_u node:$pg, node:$a, node:$b),
- (AArch64and_u node:$pg, node:$b, node:$a)]>;
+// Match `intr` with either data operand order, keeping the predicate unchanged.
+class CommPredBinPatFrags<SDPatternOperator intr>
+ : PatFrags<(ops node:$pg, node:$a, node:$b),
+ [(intr node:$pg, node:$a, node:$b),
+ (intr node:$pg, node:$b, node:$a)]>;
+def SVEAndU : CommPredBinPatFrags<AArch64and_u>;
+def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>;
+def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>;
+def SVEEorU : CommPredBinPatFrags<AArch64eor_u>;
+def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>;
+def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>;
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
@@ -799,6 +807,92 @@ let Predicates = [HasSVE_or_SME] in {
defm : SVEAndNotPats<nxv4i32, nxv4i1>;
defm : SVEAndNotPats<nxv2i64, nxv2i1>;
+ multiclass SVEAndXorPats<ValueType VT, ValueType PT> {
+ // Lanes inactive in EOR may be replaced by a & ~b in result.
+ def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)),
+ (BIC_ZZZ $a, $b)>;
+
+ // Inactive lanes in AND or EOR may be replaced by a & ~b.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for normal LLVM XOR.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (xor VT:$a, VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Matching predicate ensures that the active lanes in EOR can be
+ // replaced by a & ~b.
+ def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
+ (VT (SVEEorM PT:$pg, VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for _z form of EOR.
+ def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
+ (VT (SVEEorZ PT:$pg, VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AllActive predicate in EOR ensures that the active lane in the
+ // result is a & ~b.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (SVEEorM (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for _z form of EOR.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (SVEEorZ (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEAndXorPats<nxv16i8, nxv16i1>;
+ defm : SVEAndXorPats<nxv8i16, nxv8i1>;
+ defm : SVEAndXorPats<nxv4i32, nxv4i1>;
+ defm : SVEAndXorPats<nxv2i64, nxv2i1>;
+
+ multiclass SVEXorAndPats<ValueType VT, ValueType PT> {
+ // Lanes inactive in AND may be replaced by a & ~b in result.
+ def : Pat<(xor VT:$a, (SVEAndU (PT srcvalue), VT:$a, VT:$b)),
+ (BIC_ZZZ $a, $b)>;
+
+ // Inactive lanes in EOR or AND may be replaced by a & ~b.
+ def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a,
+ (VT (SVEAndU (PT srcvalue), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for normal LLVM AND.
+ def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, (and VT:$a, VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Matching predicate ensures that the active lanes in EOR can be
+ // replaced by a & ~b.
+ def : Pat<(VT (SVEEorU PT:$pg, VT:$a,
+ (VT (SVEAndM PT:$pg, VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for _z form of AND.
+ def : Pat<(VT (SVEEorU PT:$pg, VT:$a,
+ (VT (SVEAndZ PT:$pg, VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AllActive predicate in AND node ensures that the active lane
+ // in the result may be replaced by a & ~b.
+ def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a,
+ (VT (SVEAndM (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Similarly, for _z form of AND.
+ def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a,
+ (VT (SVEAndZ (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fallback.
+ def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)),
+ (EOR_ZZZ $a, $b)>;
+ }
+ defm : SVEXorAndPats<nxv16i8, nxv16i1>;
+ defm : SVEXorAndPats<nxv8i16, nxv8i1>;
+ defm : SVEXorAndPats<nxv4i32, nxv4i1>;
+ defm : SVEXorAndPats<nxv2i64, nxv2i1>;
+
defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
@@ -888,16 +982,21 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
- // Allow AND_U nodes to select AND-immediate instructions.
- class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
- ValueType IT, ComplexPattern CP>
- : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ // Allow AND_U and EOR_U nodes to select AND-immediate/EOR-immediate
+ // instructions.
+ multiclass SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP> {
+ def : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
(VT (splat_vector (IT (CP i64:$imm)))))),
(AND_ZI_PSEUDO $a, i64:$imm)>;
- def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
- def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
- def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
- def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+ def : Pat<(VT (SVEEorU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (EOR_ZI_PSEUDO $a, i64:$imm)>;
+ }
+ defm : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ defm : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ defm : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ defm : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll
index 9c7d88f111b0d0..4555d2c1c4b931 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll
@@ -6,8 +6,7 @@
define <vscale x 4 x i32> @and_eor_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_eor_u:
; CHECK: // %bb.0:
-; CHECK-NEXT: eor z1.d, z0.d, z1.d
-; CHECK-NEXT: and z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %eor
@@ -17,8 +16,7 @@ define <vscale x 4 x i32> @and_eor_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
define <vscale x 4 x i32> @and_u_eor_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_u_eor_u:
; CHECK: // %bb.0:
-; CHECK-NEXT: eor z1.d, z0.d, z1.d
-; CHECK-NEXT: and z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg1, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
@@ -28,8 +26,7 @@ define <vscale x 4 x i32> @and_u_eor_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1>
define <vscale x 4 x i32> @and_u_xor(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_u_xor:
; CHECK: // %bb.0:
-; CHECK-NEXT: eor z1.d, z0.d, z1.d
-; CHECK-NEXT: and z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%xor = xor <vscale x 4 x i32> %a, %b
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %xor)
@@ -39,9 +36,7 @@ define <vscale x 4 x i32> @and_u_xor(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
define <vscale x 4 x i32> @and_u_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_u_eor_m:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, z0.d
-; CHECK-NEXT: eor z2.s, p0/m, z2.s, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
@@ -51,8 +46,7 @@ define <vscale x 4 x i32> @and_u_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
define <vscale x 4 x i32> @and_u_eor_m_commuted(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_u_eor_m_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: eor z1.s, p0/m, z1.s, z0.s
-; CHECK-NEXT: and z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %a)
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
@@ -83,13 +77,20 @@ define <vscale x 4 x i32> @and_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
ret <vscale x 4 x i32> %and
}
+define <vscale x 4 x i32> @and_u_eor_z(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: and_u_eor_z:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
+ ret <vscale x 4 x i32> %and
+}
+
define <vscale x 4 x i32> @and_u_eor_m_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_u_eor_m_all_active:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, z0.d
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: eor z2.s, p0/m, z2.s, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
@@ -166,11 +167,20 @@ define <vscale x 4 x i32> @and_m_eor_m_all_active(<vscale x 4 x i1> %pg, <vscale
ret <vscale x 4 x i32> %and
}
+define <vscale x 4 x i32> @and_u_eor_z_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: and_u_eor_z_all_active:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor)
+ ret <vscale x 4 x i32> %and
+}
+
define <vscale x 16 x i8> @and_u_xor_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: and_u_xor_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: eor z1.d, z0.d, z1.d
-; CHECK-NEXT: and z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%xor = xor <vscale x 16 x i8> %a, %b
%and = tail call <vscale x 16 x i8> @llvm.aarch64.sve.and.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %xor)
@@ -218,6 +228,7 @@ declare <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1>, <v
declare <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; SME: {{.*}}
; SVE: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll
index 9ca1260dc7c405..8f52d72db262ea 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll
@@ -6,8 +6,7 @@
define <vscale x 4 x i32> @xor_and_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: xor_and_u:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z1.d, z0.d, z1.d
-; CHECK-NEXT: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%xor = xor <vscale x 4 x i32> %a, %and
@@ -17,8 +16,7 @@ define <vscale x 4 x i32> @xor_and_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
define <vscale x 4 x i32> @eor_u_and_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and_u:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z1.d, z0.d, z1.d
-; CHECK-NEXT: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg1, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
@@ -28,8 +26,7 @@ define <vscale x 4 x i32> @eor_u_and_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1>
define <vscale x 4 x i32> @eor_u_and(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z1.d, z0.d, z1.d
-; CHECK-NEXT: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = and <vscale x 4 x i32> %a, %b
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
@@ -39,9 +36,7 @@ define <vscale x 4 x i32> @eor_u_and(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
define <vscale x 4 x i32> @eor_u_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and_m:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, z0.d
-; CHECK-NEXT: and z2.s, p0/m, z2.s, z1.s
-; CHECK-NEXT: eor z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
@@ -51,14 +46,23 @@ define <vscale x 4 x i32> @eor_u_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
define <vscale x 4 x i32> @eor_u_and_m_commuted(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and_m_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z1.s, p0/m, z1.s, z0.s
-; CHECK-NEXT: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %a)
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
ret <vscale x 4 x i32> %eor
}
+define <vscale x 4 x i32> @eor_u_and_z(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: eor_u_and_z:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
+ ret <vscale x 4 x i32> %eor
+}
+
define <vscale x 4 x i32> @eor_u_and_m_pred_mismatch(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and_m_pred_mismatch:
; CHECK: // %bb.0:
@@ -86,10 +90,7 @@ define <vscale x 4 x i32> @xor_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %
define <vscale x 4 x i32> @eor_u_and_m_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: eor_u_and_m_all_active:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z2.d, z0.d
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: and z2.s, p0/m, z2.s, z1.s
-; CHECK-NEXT: eor z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
%eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
@@ -166,11 +167,20 @@ define <vscale x 4 x i32> @eor_m_and_m_all_active(<vscale x 4 x i1> %pg, <vscale
ret <vscale x 4 x i32> %eor
}
+define <vscale x 4 x i32> @eor_u_and_z_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: eor_u_and_z_all_active:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and)
+ ret <vscale x 4 x i32> %eor
+}
+
define <vscale x 16 x i8> @eor_u_and_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: eor_u_and_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z1.d, z0.d, z1.d
-; CHECK-NEXT: eor z0.d, z0.d, z1.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%and = and <vscale x 16 x i8> %a, %b
%eor = tail call <vscale x 16 x i8> @llvm.aarch64.sve.eor.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %and)
@@ -218,6 +228,7 @@ declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1>, <v
declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.eor.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; SME: {{.*}}
; SVE: {{.*}}
>From 992d7afb777755bff7f270e22365718cee2e906d Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Mon, 9 Nov 2026 09:34:01 +0000
Subject: [PATCH 2/4] fixup! [AArch64] Add patterns to combine SVE AND and XOR
to BIC
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 4 ++--
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 10 +++++-----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 19214f4469c23c..9e9be9da5eaa95 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25884,13 +25884,13 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0),
+ return DAG.getNode(AArch64ISD::AND_PRED, SDLoc(N), N->getValueType(0),
N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_eor_u:
- return DAG.getNode(AArch64ISD::EOR_U, SDLoc(N), N->getValueType(0),
+ return DAG.getNode(AArch64ISD::EOR_PRED, SDLoc(N), N->getValueType(0),
N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_orr_u:
return DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), N->getOperand(2),
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7c51d5a2ffd19d..925d27406c69c8 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -251,8 +251,8 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [
]>;
// Predicated operations with the result of inactive lanes being unspecified.
-def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>;
-def AArch64eor_u : SDNode<"AArch64ISD::EOR_U", SDT_AArch64Arith>;
+def AArch64and_p : SDNode<"AArch64ISD::AND_PRED", SDT_AArch64Arith>;
+def AArch64eor_p : SDNode<"AArch64ISD::EOR_PRED", SDT_AArch64Arith>;
def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>;
def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>;
def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>;
@@ -566,10 +566,10 @@ class CommPredBinPatFrags<SDPatternOperator intr>
: PatFrags<(ops node:$pg, node:$a, node:$b),
[(intr node:$pg, node:$a, node:$b),
(intr node:$pg, node:$b, node:$a)]>;
-def SVEAndU : CommPredBinPatFrags<AArch64and_u>;
+def SVEAndU : CommPredBinPatFrags<AArch64and_p>;
def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>;
def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>;
-def SVEEorU : CommPredBinPatFrags<AArch64eor_u>;
+def SVEEorU : CommPredBinPatFrags<AArch64eor_p>;
def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>;
def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>;
@@ -799,7 +799,7 @@ let Predicates = [HasSVE_or_SME] in {
// Fall back to register AND when no BIC pattern matches.
// AND_U allows any result in inactive lanes.
- def : Pat<(VT (AArch64and_u (PT srcvalue), VT:$a, VT:$b)),
+ def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)),
(AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
>From 04be5b5a88e63f393c8e450c5a0251dd8ef61ca6 Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Mon, 9 Nov 2026 09:38:51 +0000
Subject: [PATCH 3/4] fixup! [AArch64] Add patterns to combine SVE AND and XOR
to BIC
Reorganise placement of patterns and multiclasses.
---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 98 ++++++++-----------
llvm/lib/Target/AArch64/SVEInstrFormats.td | 29 ++++++
2 files changed, 71 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 925d27406c69c8..04995c93bb4293 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -561,16 +561,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
-// Match `intr` with either data operand order, keeping the predicate unchanged.
-class CommPredBinPatFrags<SDPatternOperator intr>
- : PatFrags<(ops node:$pg, node:$a, node:$b),
- [(intr node:$pg, node:$a, node:$b),
- (intr node:$pg, node:$b, node:$a)]>;
+// Match the specified binary op with either order of data operands.
def SVEAndU : CommPredBinPatFrags<AArch64and_p>;
-def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>;
def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>;
def SVEEorU : CommPredBinPatFrags<AArch64eor_p>;
-def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>;
def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>;
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
@@ -660,9 +654,9 @@ def AArch64famax_p : PatFrags<(ops node:$pred, node:$op1, node:$op2),
def AArch64add_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_add, add>;
def AArch64sub_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_sub, sub>;
def AArch64mul_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_mul, AArch64mul_p>;
-def AArch64and_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_and, and>;
-def AArch64orr_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_orr, or>;
-def AArch64eor_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_eor, xor>;
+def AArch64and_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_and, and>;
+def AArch64orr_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_orr, or>;
+def AArch64eor_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_eor, xor>;
def AArch64smax_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_smax, AArch64smax_p>;
def AArch64umax_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_umax, AArch64umax_p>;
def AArch64smin_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_smin, AArch64smin_p>;
@@ -767,6 +761,18 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
+ defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
+ defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
+ defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
+
+ defm ORR_ZPmZ : sve_int_bin_pred_log<0b000, "orr", AArch64orr_m1, DestructiveBinaryComm>;
+ defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>;
+ defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>;
+ defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>;
+} // End HasSVE_or_SME
+
+// Rewrites specific to selecting BIC_ZZZ.
+let Predicates = [HasSVE_or_SME] in {
multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
// Ordinary AND: NOT's undefined inactive lanes may take the value ~b.
def : Pat<(and VT:$a,
@@ -807,6 +813,26 @@ let Predicates = [HasSVE_or_SME] in {
defm : SVEAndNotPats<nxv4i32, nxv4i1>;
defm : SVEAndNotPats<nxv2i64, nxv2i1>;
+ multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT,
+ Instruction PredBic> {
+ // Preserve A in inactive lanes. NOT must cover every active AND lane.
+ def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
+ (PredBic $pg, $a, $b)>;
+ def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (PredBic $pg, $a, $b)>;
+
+ // With both predicates all-active, no inactive lanes need preserving.
+ def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>;
+ defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>;
+ defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>;
+ defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>;
+
multiclass SVEAndXorPats<ValueType VT, ValueType PT> {
// Lanes inactive in EOR may be replaced by a & ~b in result.
def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)),
@@ -824,7 +850,7 @@ let Predicates = [HasSVE_or_SME] in {
// Matching predicate ensures that the active lanes in EOR can be
// replaced by a & ~b.
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
- (VT (SVEEorM PT:$pg, VT:$a, VT:$b)))),
+ (VT (AArch64eor_m1 PT:$pg, VT:$a, VT:$b)))),
(BIC_ZZZ $a, $b)>;
// Similarly, for _z form of EOR.
@@ -835,7 +861,7 @@ let Predicates = [HasSVE_or_SME] in {
// AllActive predicate in EOR ensures that the active lane in the
// result is a & ~b.
def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (SVEEorM (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (VT (AArch64eor_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))),
(BIC_ZZZ $a, $b)>;
// Similarly, for _z form of EOR.
@@ -865,7 +891,7 @@ let Predicates = [HasSVE_or_SME] in {
// Matching predicate ensures that the active lanes in EOR can be
// replaced by a & ~b.
def : Pat<(VT (SVEEorU PT:$pg, VT:$a,
- (VT (SVEAndM PT:$pg, VT:$a, VT:$b)))),
+ (VT (AArch64and_m1 PT:$pg, VT:$a, VT:$b)))),
(BIC_ZZZ $a, $b)>;
// Similarly, for _z form of AND.
@@ -876,7 +902,7 @@ let Predicates = [HasSVE_or_SME] in {
// AllActive predicate in AND node ensures that the active lane
// in the result may be replaced by a & ~b.
def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a,
- (VT (SVEAndM (PT (SVEAllActive)), VT:$a, VT:$b)))),
+ (VT (AArch64and_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))),
(BIC_ZZZ $a, $b)>;
// Similarly, for _z form of AND.
@@ -892,35 +918,6 @@ let Predicates = [HasSVE_or_SME] in {
defm : SVEXorAndPats<nxv8i16, nxv8i1>;
defm : SVEXorAndPats<nxv4i32, nxv4i1>;
defm : SVEXorAndPats<nxv2i64, nxv2i1>;
-
- defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
- defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
- defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
-
- defm ORR_ZPmZ : sve_int_bin_pred_log<0b000, "orr", AArch64orr_m1, DestructiveBinaryComm>;
- defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>;
- defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>;
- defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>;
-
- multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT,
- Instruction PredBic> {
- // Preserve A in inactive lanes. NOT must cover every active AND lane.
- def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
- (PredBic $pg, $a, $b)>;
- def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
- (PredBic $pg, $a, $b)>;
-
- // With both predicates all-active, no inactive lanes need preserving.
- def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
- }
- defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>;
- defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>;
- defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>;
- defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>;
} // End HasSVE_or_SME
let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in {
@@ -984,19 +981,8 @@ let Predicates = [HasSVE_or_SME] in {
// Allow AND_U and EOR_U nodes to select AND-immediate/EOR-immediate
// instructions.
- multiclass SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
- ValueType IT, ComplexPattern CP> {
- def : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
- (VT (splat_vector (IT (CP i64:$imm)))))),
- (AND_ZI_PSEUDO $a, i64:$imm)>;
- def : Pat<(VT (SVEEorU (PT (SVEAnyPredicate)), (VT ZPR:$a),
- (VT (splat_vector (IT (CP i64:$imm)))))),
- (EOR_ZI_PSEUDO $a, i64:$imm)>;
- }
- defm : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
- defm : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
- defm : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
- defm : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+ defm : sve_log_dupimm_any_pred<SVEAndU, AND_ZI_PSEUDO>;
+ defm : sve_log_dupimm_any_pred<SVEEorU, EOR_ZI_PSEUDO>;
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 7e684c43e2ccfa..0b1da3f72c74f6 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -734,6 +734,12 @@ class SVE_Shift_DupImm_Any_Predicate_Pat<ValueType vt, SDPatternOperator op,
: Pat<(vt (op (pt (SVEAnyPredicate)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))),
(inst $Rn, i32:$imm)>;
+class SVE_Log_DupImm_Any_Pred_Pat<ValueType vt, SDPatternOperator op,
+ ValueType pt, ZPRRegOp zprty, ValueType it,
+ ComplexPattern cast, Instruction inst>
+: Pat<(vt (op (pt (SVEAnyPredicate)), (vt zprty:$Rn), (vt (splat_vector (it (cast i64:$imm)))))),
+ (inst $Rn, i64:$imm)>;
+
class SVE_2_Op_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, ValueType pt,
ValueType it, ComplexPattern cpx, Instruction inst>
: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Op1, (SVEDup0)),
@@ -805,6 +811,22 @@ class VSelectUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOper
], [{
return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();
}]>;
+// Same as above with a commutative operation
+class VSelectCommUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>
+: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [
+ (intrinsic node:$Pg, node:$Op1, node:$Op2),
+ (intrinsic node:$Pg, node:$Op2, node:$Op1),
+ (vselect node:$Pg, (sdnode node:$Op1, node:$Op2), node:$Op1),
+ (vselect node:$Pg, (sdnode node:$Op2, node:$Op1), node:$Op1)
+ ], [{
+ return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();
+ }]>;
+
+// Match `intr` with either data operand order, keeping the predicate unchanged.
+class CommPredBinPatFrags<SDPatternOperator intr>
+ : PatFrags<(ops node:$pg, node:$a, node:$b),
+ [(intr node:$pg, node:$a, node:$b),
+ (intr node:$pg, node:$b, node:$a)]>;
//
// Pseudo -> Instruction mappings
@@ -2189,6 +2211,13 @@ multiclass sve_int_log_imm_bic<SDPatternOperator op> {
def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>;
}
+multiclass sve_log_dupimm_any_pred<SDPatternOperator op, Instruction inst> {
+ def : SVE_Log_DupImm_Any_Pred_Pat<nxv16i8, op, nxv16i1, ZPR8, i32, SVELogicalImm8Pat, inst>;
+ def : SVE_Log_DupImm_Any_Pred_Pat<nxv8i16, op, nxv8i1, ZPR16, i32, SVELogicalImm16Pat, inst>;
+ def : SVE_Log_DupImm_Any_Pred_Pat<nxv4i32, op, nxv4i1, ZPR32, i32, SVELogicalImm32Pat, inst>;
+ def : SVE_Log_DupImm_Any_Pred_Pat<nxv2i64, op, nxv2i1, ZPR64, i64, SVELogicalImm64Pat, inst>;
+}
+
class sve_int_dup_mask_imm<string asm>
: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms),
asm, "\t$Zd, $imms",
>From 47f17f4ff0d57a00703b0174af056768e9f0ac54 Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Mon, 9 Nov 2026 09:51:53 +0000
Subject: [PATCH 4/4] fixup! [AArch64] Add patterns to combine SVE AND and XOR
to BIC
Rename CommPredBinPatFrags -> PredicatedCommutativeBinOpPatFrags to make
the name more informative.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 8 ++++----
llvm/lib/Target/AArch64/SVEInstrFormats.td | 8 ++++----
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 04995c93bb4293..0a129df2d8829a 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -562,10 +562,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(AArch64bic_node node:$op1, node:$op2)]>;
// Match the specified binary op with either order of data operands.
-def SVEAndU : CommPredBinPatFrags<AArch64and_p>;
-def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>;
-def SVEEorU : CommPredBinPatFrags<AArch64eor_p>;
-def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>;
+def SVEAndU : PredicatedCommutativeBinOpPatFrags<AArch64and_p>;
+def SVEAndZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_and_z>;
+def SVEEorU : PredicatedCommutativeBinOpPatFrags<AArch64eor_p>;
+def SVEEorZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_eor_z>;
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 0b1da3f72c74f6..37be38731c63a1 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -822,11 +822,11 @@ class VSelectCommUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPattern
return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse();
}]>;
-// Match `intr` with either data operand order, keeping the predicate unchanged.
-class CommPredBinPatFrags<SDPatternOperator intr>
+// Match `op` with either data operand order, keeping the predicate unchanged.
+class PredicatedCommutativeBinOpPatFrags<SDPatternOperator op>
: PatFrags<(ops node:$pg, node:$a, node:$b),
- [(intr node:$pg, node:$a, node:$b),
- (intr node:$pg, node:$b, node:$a)]>;
+ [(op node:$pg, node:$a, node:$b),
+ (op node:$pg, node:$b, node:$a)]>;
//
// Pseudo -> Instruction mappings
More information about the llvm-branch-commits
mailing list