[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms (PR #232526)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 23 07:50:51 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 8619ed708a5dcc30a8b267188adcbfbb2736a0a7 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 14 Oct 2026 23:22:09 +0100
Subject: [PATCH 01/10] [AArch64][llvm] Combine SVE AND-NOT operations into BIC
for _x forms
Replace NOT followed by AND with BIC for expressions such as:
return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b));
Combine ordinary AND with NOT when the NOT predicate is all-active or
its inactive values are undefined.
Also combine matching-predicate SVE AND_x-NOT pairs before lowering
discards the AND predicate. AND_x ignores inactive results, so the NOT's
inactive values do not need preserving.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++-
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++--------------
2 files changed, 57 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1cd81ba0c259..060ecf1614fcd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22134,7 +22134,21 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
+ assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
+
+ // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
+ // matter, leaving standalone NOTs available for predicated selection.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
+ continue;
+ if (Not.getOperand(1).isUndef() ||
+ isAllActivePredicate(DAG, Not.getOperand(2)))
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(1 - I), Not.getOperand(3));
+ }
+
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25442,6 +25456,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
+static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(1);
+
+ // Fold matching predicates before lowering AND_U discards its predicate.
+ // NOT's inactive lanes cannot affect AND_U's active lanes.
+ for (unsigned I = 0; I != 2; ++I) {
+ unsigned NotOperandIdx = 2 + I;
+ unsigned OtherOperandIdx = 3 - I;
+ SDValue Not = N->getOperand(NotOperandIdx);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
+ Not.getOperand(2) != Pg)
+ continue;
+
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ }
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
+ N->getOperand(3));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25661,8 +25695,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 736c0ee0171c5..bc59860c715f3 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -6,26 +6,10 @@
; Ordinary LLVM AND with SVE NOT.
define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: and_not_ptrue_all:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_ptrue_all:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_ptrue_all:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_ptrue_all:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
@@ -33,26 +17,10 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_commuted:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z1.d, z0.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_commuted:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z1.d, z0.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_commuted:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z1.d, z0.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %not, %a
ret <vscale x 4 x i32> %and
@@ -105,23 +73,10 @@ define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8
}
define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i32:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
ret <vscale x 4 x i32> %and
@@ -176,26 +131,10 @@ define <vscale x 4 x i32> @and_u_not_commuted_i32(<vscale x 4 x i1> %pg, <vscale
; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate.
define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_all_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_all_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_all_i32:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_all_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
@@ -204,8 +143,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_defined:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -215,8 +153,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z2.d, z0.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a)
@@ -249,26 +186,10 @@ define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <v
}
define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: all_not_partial_and:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: all_not_partial_and:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: all_not_partial_and:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: all_not_partial_and:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
>From b1f37b0734e623bc57529b85d7cdfd8a4aae3c43 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 12:55:10 +0100
Subject: [PATCH 02/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ------------
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 ++++++++++++++
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 060ecf1614fcd..3b728e3e64d63 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22137,18 +22137,6 @@ static SDValue performSVEAndCombine(SDNode *N,
assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
- // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
- // matter, leaving standalone NOTs available for predicated selection.
- for (unsigned I = 0; I != 2; ++I) {
- SDValue Not = N->getOperand(I);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
- continue;
- if (Not.getOperand(1).isUndef() ||
- isAllActivePredicate(DAG, Not.getOperand(2)))
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(1 - I), Not.getOperand(3));
- }
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9e56956dab730..9d92c34c7fbb7 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -753,6 +753,20 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
+ // Select BIC when NOT's inactive values are undefined or all lanes are active.
+ multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEAndNotPats<nxv16i8, nxv16i1>;
+ defm : SVEAndNotPats<nxv8i16, nxv8i1>;
+ defm : SVEAndNotPats<nxv4i32, nxv4i1>;
+ defm : SVEAndNotPats<nxv2i64, nxv2i1>;
+
defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
>From 88c71ef5e8af23cbc04cefe95f1a044afb2f1e54 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 13:43:41 +0100
Subject: [PATCH 03/10] fixup! Add tests for extra widths
---
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 63 ++++---------------
1 file changed, 12 insertions(+), 51 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index bc59860c715f3..4d8d595f242c2 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -27,46 +27,20 @@ define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x
}
define <vscale x 16 x i8> @and_not_poison_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i8:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.b, p0/m, z1.b
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i8:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.b, p0/z, z1.b
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i8:
-; SME: // %bb.0:
-; SME-NEXT: not z1.b, p0/m, z1.b
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
%and = and <vscale x 16 x i8> %a, %not
ret <vscale x 16 x i8> %and
}
define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i16:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.h, p0/m, z1.h
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i16:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.h, p0/z, z1.h
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i16:
-; SME: // %bb.0:
-; SME-NEXT: not z1.h, p0/m, z1.h
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b)
%and = and <vscale x 8 x i16> %a, %not
ret <vscale x 8 x i16> %and
@@ -83,23 +57,10 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 2 x i64> @and_not_poison_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i64:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.d, p0/m, z1.d
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i64:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.d, p0/z, z1.d
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i64:
-; SME: // %bb.0:
-; SME-NEXT: not z1.d, p0/m, z1.d
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b)
%and = and <vscale x 2 x i64> %a, %not
ret <vscale x 2 x i64> %and
>From 33a2434a36203e546b64bad6e5514b3901a0e195 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 16:27:26 +0100
Subject: [PATCH 04/10] fixup! Replace performSVEAndNotIntrinsicCombine() with
tablegen
Move AND_U and NOT matching to TableGen, using a shared pattern fragment
for both operand orders. Retain register and immediate AND selection
for other AND_U operations.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 38 ++++++++++------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 ++++++++++++++++++-
2 files changed, 67 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3b728e3e64d63..011c074992c99 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22134,9 +22134,7 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
- assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25444,22 +25442,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
SDValue Pg = N->getOperand(1);
- // Fold matching predicates before lowering AND_U discards its predicate.
- // NOT's inactive lanes cannot affect AND_U's active lanes.
- for (unsigned I = 0; I != 2; ++I) {
- unsigned NotOperandIdx = 2 + I;
- unsigned OtherOperandIdx = 3 - I;
- SDValue Not = N->getOperand(NotOperandIdx);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
- Not.getOperand(2) != Pg)
+ // Preserve matching predicates so TableGen can select:
+ // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
+ for (unsigned I = 2; I != 4; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
continue;
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ SDValue NotPg = Not.getOperand(2);
+ if (NotPg != Pg)
+ continue;
+
+ // We can discard AND_U's predicate if NOT's inactive lanes are
+ // undefined or NOT's predicate is all-active: ordinary AND-NOT
+ // patterns can select BIC.
+ SDValue Inactive = Not.getOperand(1);
+ if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg))
+ continue;
+
+ // Keep AND_U unchanged so TableGen can match the predicates.
+ return SDValue();
}
+
+ // Otherwise expose an ordinary AND so generic DAG combines can simplify
+ // it, for example:
+ // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
}
@@ -25683,7 +25693,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return performSVEAndNotIntrinsicCombine(N, DAG);
+ return performSVEAndUIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9d92c34c7fbb7..26c88729e1bdf 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -559,6 +559,11 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
+// AND_U permits either operand order.
+def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
+ [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b),
+ (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>;
+
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
(and node:$pg, (and node:$op1, node:$op2))]>;
@@ -753,14 +758,40 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
- // Select BIC when NOT's inactive values are undefined or all lanes are active.
multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ // Ordinary AND: NOT's undefined inactive lanes may take the value ~b.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
(BIC_ZZZ $a, $b)>;
+ // Ordinary AND: an all-active NOT produces ~b in every lane.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
(BIC_ZZZ $a, $b)>;
+
+ // AND_U: matching predicates ensure NOT produces ~b in every active AND lane.
+ def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: NOT's undefined inactive lanes may take the value ~b,
+ // even when the predicates differ.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fall back to register AND when no BIC pattern matches.
+ // AND_U allows any result in inactive lanes.
+ def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
+ (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
@@ -825,6 +856,17 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
+ // Allow retained AND_U intrinsics to select AND-immediate instructions.
+ class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP>
+ : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (AND_ZI_PSEUDO $a, i64:$imm)>;
+ def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From e2a1a478a894d3ebf328360413874bd858a99e18 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:36:25 +0100
Subject: [PATCH 05/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the AND_U pattern for NOT with undefined inactive lanes and an arbitrary predicate. Keep the original fix focused; ordinary cases still lower to AND and use the existing AND-NOT patterns.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ------
1 file changed, 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 26c88729e1bdf..af65a5d39b31d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -773,12 +773,6 @@ let Predicates = [HasSVE_or_SME] in {
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
- // AND_U: NOT's undefined inactive lanes may take the value ~b,
- // even when the predicates differ.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
// AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
>From 1fec464223a9e51762fac61a7643d14cd1c695c5 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:44:55 +0100
Subject: [PATCH 06/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the extra AND_U patterns for all-active NOT, ordinary LLVM NOT
and register AND fallback. The C++ helper lowers ordinary cases to
AND and retains matching-predicate AND-NOT pairs for the remaining BIC
pattern.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index af65a5d39b31d..8ffdb8cb9c05f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -772,20 +772,6 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
-
- // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
- // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
- (BIC_ZZZ $a, $b)>;
-
- // Fall back to register AND when no BIC pattern matches.
- // AND_U allows any result in inactive lanes.
- def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
- (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
>From 5a7397bed853d4390faf98b7286a7de353cd8bd0 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 17:23:16 +0100
Subject: [PATCH 07/10] fixup! Remove patterns
Remove these patterns, as they're no longer needed since I've
added the C++ code in performSVEAndUIntrinsicCombine()
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 8ffdb8cb9c05f..e2dc297fd062c 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -836,17 +836,6 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
- // Allow retained AND_U intrinsics to select AND-immediate instructions.
- class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
- ValueType IT, ComplexPattern CP>
- : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
- (VT (splat_vector (IT (CP i64:$imm)))))),
- (AND_ZI_PSEUDO $a, i64:$imm)>;
- def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
- def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
- def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
- def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
-
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From 2121a9d8ea0473dedc63df4367c20c2a1e349e5b Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 18:49:10 +0100
Subject: [PATCH 08/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Lower AND_U to an AArch64 target DAG node and match that node in TableGen. Keep the predicate for matching AND-NOT pairs, and combine other cases to ordinary AND so generic simplifications remain available. Name the data operands and iterate over them directly.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 17 ++++++++++-------
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 7 ++++---
2 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 011c074992c99..91297239c5a2a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25442,13 +25442,14 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
- SDValue Pg = N->getOperand(1);
+static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(0);
+ SDValue A = N->getOperand(1);
+ SDValue B = N->getOperand(2);
// Preserve matching predicates so TableGen can select:
// and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
- for (unsigned I = 2; I != 4; ++I) {
- SDValue Not = N->getOperand(I);
+ for (SDValue Not : {A, B}) {
if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
continue;
@@ -25470,8 +25471,7 @@ static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
// Otherwise expose an ordinary AND so generic DAG combines can simplify
// it, for example:
// and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B);
}
static SDValue performIntrinsicCombine(SDNode *N,
@@ -25693,7 +25693,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return performSVEAndUIntrinsicCombine(N, DAG);
+ return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
@@ -32020,6 +32021,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performORCombine(N, DCI);
case ISD::AND:
return performANDCombine(N, DCI);
+ case AArch64ISD::AND_U:
+ return performSVEAndUCombine(N, DAG);
case ISD::FADD:
return performFADDCombine(N, DCI);
case AArch64ISD::FADD_PRED:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index e2dc297fd062c..6b50de7abe7d6 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -251,6 +251,7 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [
]>;
// Predicated operations with the result of inactive lanes being unspecified.
+def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>;
def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>;
def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>;
def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>;
@@ -559,10 +560,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
-// AND_U permits either operand order.
+// Match AND_U with either data operand order, keeping the predicate unchanged.
def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
- [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b),
- (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>;
+ [(AArch64and_u node:$pg, node:$a, node:$b),
+ (AArch64and_u node:$pg, node:$b, node:$a)]>;
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
>From 54c642597496dae9d14b311a74a8016a783e4e4d Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 23 Oct 2026 15:28:12 +0100
Subject: [PATCH 09/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Keep AND_U as an AArch64 target node until instruction selection. Remove the C++ combine and restore the TableGen patterns for BIC, register AND and immediate AND selection.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 34 -------------------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 31 +++++++++++++++++
2 files changed, 31 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 91297239c5a2a..b26f1f3f6b8bd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25442,38 +25442,6 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) {
- SDValue Pg = N->getOperand(0);
- SDValue A = N->getOperand(1);
- SDValue B = N->getOperand(2);
-
- // Preserve matching predicates so TableGen can select:
- // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
- for (SDValue Not : {A, B}) {
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
- continue;
-
- SDValue NotPg = Not.getOperand(2);
- if (NotPg != Pg)
- continue;
-
- // We can discard AND_U's predicate if NOT's inactive lanes are
- // undefined or NOT's predicate is all-active: ordinary AND-NOT
- // patterns can select BIC.
- SDValue Inactive = Not.getOperand(1);
- if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg))
- continue;
-
- // Keep AND_U unchanged so TableGen can match the predicates.
- return SDValue();
- }
-
- // Otherwise expose an ordinary AND so generic DAG combines can simplify
- // it, for example:
- // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B);
-}
-
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -32021,8 +31989,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performORCombine(N, DCI);
case ISD::AND:
return performANDCombine(N, DCI);
- case AArch64ISD::AND_U:
- return performSVEAndUCombine(N, DAG);
case ISD::FADD:
return performFADDCombine(N, DCI);
case AArch64ISD::FADD_PRED:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 6b50de7abe7d6..61b605a2cdb68 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -773,6 +773,26 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
+
+ // AND_U: NOT's undefined inactive lanes may take the value ~b,
+ // even when the predicates differ.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fall back to register AND when no BIC pattern matches.
+ // AND_U allows any result in inactive lanes.
+ def : Pat<(VT (AArch64and_u (PT srcvalue), VT:$a, VT:$b)),
+ (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
@@ -837,6 +857,17 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
+ // Allow AND_U nodes to select AND-immediate instructions.
+ class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP>
+ : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (AND_ZI_PSEUDO $a, i64:$imm)>;
+ def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From 00eaea36a9a2200ab42db7318c41074068220c82 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 23 Oct 2026 15:45:24 +0100
Subject: [PATCH 10/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Check BIC selection for AND_U with poison NOT passthrough and different predicates. Cover the pattern independently of the matching-predicate case.
---
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 21 ++++---------------
1 file changed, 4 insertions(+), 17 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 4d8d595f242c2..13b4baa8c0554 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -124,23 +124,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4
; Poison NOT passthrough permits BIC even when the predicates differ.
define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg, <vscale x 4 x i1> %not_pg) {
-; SVE-LABEL: different_predicates_poison:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p1/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: different_predicates_poison:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p1/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: different_predicates_poison:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p1/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: different_predicates_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %not_pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
More information about the llvm-branch-commits
mailing list