[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms (PR #232526)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 22 08:52:28 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 7f9a45470a0b53261ef7732f5c290b8bfac9901f Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 14 Oct 2026 23:22:09 +0100
Subject: [PATCH 1/6] [AArch64][llvm] Combine SVE AND-NOT operations into BIC
for _x forms
Replace NOT followed by AND with BIC for expressions such as:
return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b));
Combine ordinary AND with NOT when the NOT predicate is all-active or
its inactive values are undefined.
Also combine matching-predicate SVE AND_x-NOT pairs before lowering
discards the AND predicate. AND_x ignores inactive results, so the NOT's
inactive values do not need preserving.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++-
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++--------------
2 files changed, 57 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1cd81ba0c259..060ecf1614fcd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22134,7 +22134,21 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
+ assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
+
+ // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
+ // matter, leaving standalone NOTs available for predicated selection.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
+ continue;
+ if (Not.getOperand(1).isUndef() ||
+ isAllActivePredicate(DAG, Not.getOperand(2)))
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(1 - I), Not.getOperand(3));
+ }
+
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25442,6 +25456,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
+static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(1);
+
+ // Fold matching predicates before lowering AND_U discards its predicate.
+ // NOT's inactive lanes cannot affect AND_U's active lanes.
+ for (unsigned I = 0; I != 2; ++I) {
+ unsigned NotOperandIdx = 2 + I;
+ unsigned OtherOperandIdx = 3 - I;
+ SDValue Not = N->getOperand(NotOperandIdx);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
+ Not.getOperand(2) != Pg)
+ continue;
+
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ }
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
+ N->getOperand(3));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25661,8 +25695,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 8eac86cb93479..dd06c3d38c54d 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -6,26 +6,10 @@
; Ordinary LLVM AND with SVE NOT.
define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: and_not_ptrue_all:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_ptrue_all:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_ptrue_all:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_ptrue_all:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
@@ -33,49 +17,20 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_commuted:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z1.d, z0.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_commuted:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z1.d, z0.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_commuted:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z1.d, z0.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %not, %a
ret <vscale x 4 x i32> %and
}
define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i32:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
ret <vscale x 4 x i32> %and
@@ -107,26 +62,10 @@ define <vscale x 4 x i32> @and_u_not_commuted_i32(<vscale x 4 x i1> %pg, <vscale
; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate.
define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_all_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_all_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_all_i32:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_all_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
@@ -135,8 +74,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_defined:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -146,8 +84,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z2.d, z0.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a)
@@ -155,26 +92,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: all_not_partial_and:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: all_not_partial_and:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: all_not_partial_and:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: all_not_partial_and:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
>From 93831a21b6b8fd93719c16a2ae98a2a233de8e0a Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 12:55:10 +0100
Subject: [PATCH 2/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ------------
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 ++++++++++++++
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 060ecf1614fcd..3b728e3e64d63 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22137,18 +22137,6 @@ static SDValue performSVEAndCombine(SDNode *N,
assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
- // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
- // matter, leaving standalone NOTs available for predicated selection.
- for (unsigned I = 0; I != 2; ++I) {
- SDValue Not = N->getOperand(I);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
- continue;
- if (Not.getOperand(1).isUndef() ||
- isAllActivePredicate(DAG, Not.getOperand(2)))
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(1 - I), Not.getOperand(3));
- }
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9e56956dab730..9d92c34c7fbb7 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -753,6 +753,20 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
+ // Select BIC when NOT's inactive values are undefined or all lanes are active.
+ multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEAndNotPats<nxv16i8, nxv16i1>;
+ defm : SVEAndNotPats<nxv8i16, nxv8i1>;
+ defm : SVEAndNotPats<nxv4i32, nxv4i1>;
+ defm : SVEAndNotPats<nxv2i64, nxv2i1>;
+
defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
>From 9722027e70997cd9c4ac1891b2cc1c436d5c8b16 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 13:43:41 +0100
Subject: [PATCH 3/6] fixup! Add tests for extra widths
---
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 30 +++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index dd06c3d38c54d..1a1b53a90a746 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -26,6 +26,26 @@ define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x
ret <vscale x 4 x i32> %and
}
+define <vscale x 16 x i8> @and_not_poison_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i1> %pg) {
+; CHECK-LABEL: and_not_poison_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
+ %and = and <vscale x 16 x i8> %a, %not
+ ret <vscale x 16 x i8> %and
+}
+
+define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i1> %pg) {
+; CHECK-LABEL: and_not_poison_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b)
+ %and = and <vscale x 8 x i16> %a, %not
+ ret <vscale x 8 x i16> %and
+}
+
define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: and_not_poison_i32:
; CHECK: // %bb.0:
@@ -36,6 +56,16 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4
ret <vscale x 4 x i32> %and
}
+define <vscale x 2 x i64> @and_not_poison_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i1> %pg) {
+; CHECK-LABEL: and_not_poison_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
+ %not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b)
+ %and = and <vscale x 2 x i64> %a, %not
+ ret <vscale x 2 x i64> %and
+}
+
; SVE AND_U with ordinary LLVM NOT.
define <vscale x 4 x i32> @and_u_not_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
>From cad17f3a124e8d2880e6f040b44e6294793533ae Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 16:27:26 +0100
Subject: [PATCH 4/6] fixup! Replace performSVEAndNotIntrinsicCombine() with
tablegen
Move AND_U and NOT matching to TableGen, using a shared pattern fragment
for both operand orders. Retain register and immediate AND selection
for other AND_U operations.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 38 ++++++++++------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 ++++++++++++++++++-
2 files changed, 67 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3b728e3e64d63..011c074992c99 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22134,9 +22134,7 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
- assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25444,22 +25442,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
SDValue Pg = N->getOperand(1);
- // Fold matching predicates before lowering AND_U discards its predicate.
- // NOT's inactive lanes cannot affect AND_U's active lanes.
- for (unsigned I = 0; I != 2; ++I) {
- unsigned NotOperandIdx = 2 + I;
- unsigned OtherOperandIdx = 3 - I;
- SDValue Not = N->getOperand(NotOperandIdx);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
- Not.getOperand(2) != Pg)
+ // Preserve matching predicates so TableGen can select:
+ // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
+ for (unsigned I = 2; I != 4; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
continue;
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ SDValue NotPg = Not.getOperand(2);
+ if (NotPg != Pg)
+ continue;
+
+ // We can discard AND_U's predicate if NOT's inactive lanes are
+ // undefined or NOT's predicate is all-active: ordinary AND-NOT
+ // patterns can select BIC.
+ SDValue Inactive = Not.getOperand(1);
+ if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg))
+ continue;
+
+ // Keep AND_U unchanged so TableGen can match the predicates.
+ return SDValue();
}
+
+ // Otherwise expose an ordinary AND so generic DAG combines can simplify
+ // it, for example:
+ // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
}
@@ -25683,7 +25693,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return performSVEAndNotIntrinsicCombine(N, DAG);
+ return performSVEAndUIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9d92c34c7fbb7..26c88729e1bdf 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -559,6 +559,11 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
+// AND_U permits either operand order.
+def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
+ [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b),
+ (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>;
+
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
(and node:$pg, (and node:$op1, node:$op2))]>;
@@ -753,14 +758,40 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
- // Select BIC when NOT's inactive values are undefined or all lanes are active.
multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ // Ordinary AND: NOT's undefined inactive lanes may take the value ~b.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
(BIC_ZZZ $a, $b)>;
+ // Ordinary AND: an all-active NOT produces ~b in every lane.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
(BIC_ZZZ $a, $b)>;
+
+ // AND_U: matching predicates ensure NOT produces ~b in every active AND lane.
+ def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: NOT's undefined inactive lanes may take the value ~b,
+ // even when the predicates differ.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fall back to register AND when no BIC pattern matches.
+ // AND_U allows any result in inactive lanes.
+ def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
+ (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
@@ -825,6 +856,17 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
+ // Allow retained AND_U intrinsics to select AND-immediate instructions.
+ class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP>
+ : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (AND_ZI_PSEUDO $a, i64:$imm)>;
+ def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From ca00132647c0ad5fa2a6653bdf8f0b300ff546f8 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:36:25 +0100
Subject: [PATCH 5/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the AND_U pattern for NOT with undefined inactive lanes and an arbitrary predicate. Keep the original fix focused; ordinary cases still lower to AND and use the existing AND-NOT patterns.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ------
1 file changed, 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 26c88729e1bdf..af65a5d39b31d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -773,12 +773,6 @@ let Predicates = [HasSVE_or_SME] in {
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
- // AND_U: NOT's undefined inactive lanes may take the value ~b,
- // even when the predicates differ.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
// AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
>From 96015be30d9e1c07422e142d9e0e3a472e815e31 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:44:55 +0100
Subject: [PATCH 6/6] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the extra AND_U patterns for all-active NOT, ordinary LLVM NOT
and register AND fallback. The C++ helper lowers ordinary cases to
AND and retains matching-predicate AND-NOT pairs for the remaining BIC
pattern.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index af65a5d39b31d..8ffdb8cb9c05f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -772,20 +772,6 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
-
- // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
- // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
- (BIC_ZZZ $a, $b)>;
-
- // Fall back to register AND when no BIC pattern matches.
- // AND_U allows any result in inactive lanes.
- def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
- (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
More information about the llvm-branch-commits
mailing list