[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms (PR #232526)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 26 03:19:43 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 655f40cdb4224f0ff0b0604c430f654926fbc26e Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 14 Oct 2026 23:22:09 +0100
Subject: [PATCH 01/10] [AArch64][llvm] Combine SVE AND-NOT operations into BIC
for _x forms
Replace NOT followed by AND with BIC for expressions such as:
return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b));
Combine ordinary AND with NOT when the NOT predicate is all-active or
its inactive values are undefined.
Also combine matching-predicate SVE AND_x-NOT pairs before lowering
discards the AND predicate. AND_x ignores inactive results, so the NOT's
inactive values do not need preserving.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 37 +++++-
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++--------------
2 files changed, 57 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dbdc98611f5340..bd915200219d12 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22364,7 +22364,21 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
+ assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
+
+ // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
+ // matter, leaving standalone NOTs available for predicated selection.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
+ continue;
+ if (Not.getOperand(1).isUndef() ||
+ isAllActivePredicate(DAG, Not.getOperand(2)))
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(1 - I), Not.getOperand(3));
+ }
+
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25623,6 +25637,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
+static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(1);
+
+ // Fold matching predicates before lowering AND_U discards its predicate.
+ // NOT's inactive lanes cannot affect AND_U's active lanes.
+ for (unsigned I = 0; I != 2; ++I) {
+ unsigned NotOperandIdx = 2 + I;
+ unsigned OtherOperandIdx = 3 - I;
+ SDValue Not = N->getOperand(NotOperandIdx);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
+ Not.getOperand(2) != Pg)
+ continue;
+
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ }
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
+ N->getOperand(3));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25842,8 +25876,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 736c0ee0171c54..bc59860c715f39 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -6,26 +6,10 @@
; Ordinary LLVM AND with SVE NOT.
define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: and_not_ptrue_all:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_ptrue_all:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_ptrue_all:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_ptrue_all:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
@@ -33,26 +17,10 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_commuted:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z1.d, z0.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_commuted:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z1.d, z0.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_commuted:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z1.d, z0.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %not, %a
ret <vscale x 4 x i32> %and
@@ -105,23 +73,10 @@ define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8
}
define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i32:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
ret <vscale x 4 x i32> %and
@@ -176,26 +131,10 @@ define <vscale x 4 x i32> @and_u_not_commuted_i32(<vscale x 4 x i1> %pg, <vscale
; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate.
define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_all_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_all_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_all_i32:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_all_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
@@ -204,8 +143,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_defined:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -215,8 +153,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z2.d, z0.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a)
@@ -249,26 +186,10 @@ define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <v
}
define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: all_not_partial_and:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: all_not_partial_and:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: all_not_partial_and:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: all_not_partial_and:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
>From 85888670ae88764747be627c1351b30d8bf2da8a Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 12:55:10 +0100
Subject: [PATCH 02/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ------------
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 ++++++++++++++
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bd915200219d12..540da867dc9678 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22367,18 +22367,6 @@ static SDValue performSVEAndCombine(SDNode *N,
assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
- // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
- // matter, leaving standalone NOTs available for predicated selection.
- for (unsigned I = 0; I != 2; ++I) {
- SDValue Not = N->getOperand(I);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
- continue;
- if (Not.getOperand(1).isUndef() ||
- isAllActivePredicate(DAG, Not.getOperand(2)))
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(1 - I), Not.getOperand(3));
- }
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index a3a4fdef30ef1b..958835f1858608 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -753,6 +753,20 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
+ // Select BIC when NOT's inactive values are undefined or all lanes are active.
+ multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ def : Pat<(and VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEAndNotPats<nxv16i8, nxv16i1>;
+ defm : SVEAndNotPats<nxv8i16, nxv8i1>;
+ defm : SVEAndNotPats<nxv4i32, nxv4i1>;
+ defm : SVEAndNotPats<nxv2i64, nxv2i1>;
+
defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>;
defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">;
defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>;
>From f647b2e2af470675fe5802e3bb5d928ea0de3987 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 13:43:41 +0100
Subject: [PATCH 03/10] fixup! Add tests for extra widths
---
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 63 ++++---------------
1 file changed, 12 insertions(+), 51 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index bc59860c715f39..4d8d595f242c24 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -27,46 +27,20 @@ define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x
}
define <vscale x 16 x i8> @and_not_poison_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i8:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.b, p0/m, z1.b
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i8:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.b, p0/z, z1.b
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i8:
-; SME: // %bb.0:
-; SME-NEXT: not z1.b, p0/m, z1.b
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
%and = and <vscale x 16 x i8> %a, %not
ret <vscale x 16 x i8> %and
}
define <vscale x 8 x i16> @and_not_poison_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i16:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.h, p0/m, z1.h
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i16:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.h, p0/z, z1.h
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i16:
-; SME: // %bb.0:
-; SME-NEXT: not z1.h, p0/m, z1.h
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b)
%and = and <vscale x 8 x i16> %a, %not
ret <vscale x 8 x i16> %and
@@ -83,23 +57,10 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 2 x i64> @and_not_poison_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i64:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.d, p0/m, z1.d
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i64:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.d, p0/z, z1.d
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i64:
-; SME: // %bb.0:
-; SME-NEXT: not z1.d, p0/m, z1.d
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b)
%and = and <vscale x 2 x i64> %a, %not
ret <vscale x 2 x i64> %and
>From 4253f8737eca97a87063aa2952ab7884969f245c Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 16:27:26 +0100
Subject: [PATCH 04/10] fixup! Replace performSVEAndNotIntrinsicCombine() with
tablegen
Move AND_U and NOT matching to TableGen, using a shared pattern fragment
for both operand orders. Retain register and immediate AND selection
for other AND_U operations.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 38 ++++++++++------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 44 ++++++++++++++++++-
2 files changed, 67 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 540da867dc9678..b0d8dda0a6d668 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22364,9 +22364,7 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
- assert(N->getOpcode() == ISD::AND && "Expected AND instruction");
SelectionDAG &DAG = DCI.DAG;
-
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25625,22 +25623,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
SDValue Pg = N->getOperand(1);
- // Fold matching predicates before lowering AND_U discards its predicate.
- // NOT's inactive lanes cannot affect AND_U's active lanes.
- for (unsigned I = 0; I != 2; ++I) {
- unsigned NotOperandIdx = 2 + I;
- unsigned OtherOperandIdx = 3 - I;
- SDValue Not = N->getOperand(NotOperandIdx);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
- Not.getOperand(2) != Pg)
+ // Preserve matching predicates so TableGen can select:
+ // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
+ for (unsigned I = 2; I != 4; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
continue;
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ SDValue NotPg = Not.getOperand(2);
+ if (NotPg != Pg)
+ continue;
+
+ // We can discard AND_U's predicate if NOT's inactive lanes are
+ // undefined or NOT's predicate is all-active: ordinary AND-NOT
+ // patterns can select BIC.
+ SDValue Inactive = Not.getOperand(1);
+ if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg))
+ continue;
+
+ // Keep AND_U unchanged so TableGen can match the predicates.
+ return SDValue();
}
+
+ // Otherwise expose an ordinary AND so generic DAG combines can simplify
+ // it, for example:
+ // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
}
@@ -25864,7 +25874,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return performSVEAndNotIntrinsicCombine(N, DAG);
+ return performSVEAndUIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 958835f1858608..3204884ee66915 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -559,6 +559,11 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
+// AND_U permits either operand order.
+def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
+ [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b),
+ (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>;
+
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
(and node:$pg, (and node:$op1, node:$op2))]>;
@@ -753,14 +758,40 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>;
defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>;
- // Select BIC when NOT's inactive values are undefined or all lanes are active.
multiclass SVEAndNotPats<ValueType VT, ValueType PT> {
+ // Ordinary AND: NOT's undefined inactive lanes may take the value ~b.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))),
(BIC_ZZZ $a, $b)>;
+ // Ordinary AND: an all-active NOT produces ~b in every lane.
def : Pat<(and VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))),
(BIC_ZZZ $a, $b)>;
+
+ // AND_U: matching predicates ensure NOT produces ~b in every active AND lane.
+ def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: NOT's undefined inactive lanes may take the value ~b,
+ // even when the predicates differ.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fall back to register AND when no BIC pattern matches.
+ // AND_U allows any result in inactive lanes.
+ def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
+ (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
@@ -825,6 +856,17 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
+ // Allow retained AND_U intrinsics to select AND-immediate instructions.
+ class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP>
+ : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (AND_ZI_PSEUDO $a, i64:$imm)>;
+ def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From 061db66e1f75f92e8894f5f3569eff2ec575d886 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:36:25 +0100
Subject: [PATCH 05/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the AND_U pattern for NOT with undefined inactive lanes and an arbitrary predicate. Keep the original fix focused; ordinary cases still lower to AND and use the existing AND-NOT patterns.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ------
1 file changed, 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 3204884ee66915..eba913eb05762d 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -773,12 +773,6 @@ let Predicates = [HasSVE_or_SME] in {
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
- // AND_U: NOT's undefined inactive lanes may take the value ~b,
- // even when the predicates differ.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
// AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
>From 8cf58a119ef3dfcb317fb809d7213cf983a1ef64 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 16:44:55 +0100
Subject: [PATCH 06/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Remove the extra AND_U patterns for all-active NOT, ordinary LLVM NOT
and register AND fallback. The C++ helper lowers ordinary cases to
AND and retains matching-predicate AND-NOT pairs for the remaining BIC
pattern.
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index eba913eb05762d..005a2e795a4215 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -772,20 +772,6 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
-
- // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
- (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
- (BIC_ZZZ $a, $b)>;
-
- // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
- def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
- (BIC_ZZZ $a, $b)>;
-
- // Fall back to register AND when no BIC pattern matches.
- // AND_U allows any result in inactive lanes.
- def : Pat<(VT (int_aarch64_sve_and_u (PT srcvalue), VT:$a, VT:$b)),
- (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
>From 16aaf5244136bd772cfd3587cd03bc6dc2ef8302 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 17:23:16 +0100
Subject: [PATCH 07/10] fixup! Remove patterns
Remove these patterns, as they're no longer needed since I've
added the C++ code in performSVEAndUIntrinsicCombine()
---
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 005a2e795a4215..9fb57945a06bf8 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -836,17 +836,6 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
- // Allow retained AND_U intrinsics to select AND-immediate instructions.
- class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
- ValueType IT, ComplexPattern CP>
- : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
- (VT (splat_vector (IT (CP i64:$imm)))))),
- (AND_ZI_PSEUDO $a, i64:$imm)>;
- def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
- def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
- def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
- def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
-
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From af73492a4e1e7afb198bd7cce0296bd89419d698 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 22 Oct 2026 18:49:10 +0100
Subject: [PATCH 08/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Lower AND_U to an AArch64 target DAG node and match that node in TableGen. Keep the predicate for matching AND-NOT pairs, and combine other cases to ordinary AND so generic simplifications remain available. Name the data operands and iterate over them directly.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 17 ++++++++++-------
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 7 ++++---
2 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b0d8dda0a6d668..02a7bb7001194d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25623,13 +25623,14 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
- SDValue Pg = N->getOperand(1);
+static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(0);
+ SDValue A = N->getOperand(1);
+ SDValue B = N->getOperand(2);
// Preserve matching predicates so TableGen can select:
// and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
- for (unsigned I = 2; I != 4; ++I) {
- SDValue Not = N->getOperand(I);
+ for (SDValue Not : {A, B}) {
if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
continue;
@@ -25651,8 +25652,7 @@ static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
// Otherwise expose an ordinary AND so generic DAG combines can simplify
// it, for example:
// and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B);
}
static SDValue performIntrinsicCombine(SDNode *N,
@@ -25874,7 +25874,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return performSVEAndUIntrinsicCombine(N, DAG);
+ return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2), N->getOperand(3));
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
@@ -32201,6 +32202,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performORCombine(N, DCI);
case ISD::AND:
return performANDCombine(N, DCI);
+ case AArch64ISD::AND_U:
+ return performSVEAndUCombine(N, DAG);
case ISD::FADD:
return performFADDCombine(N, DCI);
case AArch64ISD::FADD_PRED:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9fb57945a06bf8..099edf459391c9 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -251,6 +251,7 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [
]>;
// Predicated operations with the result of inactive lanes being unspecified.
+def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>;
def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>;
def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>;
def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>;
@@ -559,10 +560,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
(and node:$op1, (xor node:$op2, (SVEAllActive))),
(AArch64bic_node node:$op1, node:$op2)]>;
-// AND_U permits either operand order.
+// Match AND_U with either data operand order, keeping the predicate unchanged.
def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b),
- [(int_aarch64_sve_and_u node:$pg, node:$a, node:$b),
- (int_aarch64_sve_and_u node:$pg, node:$b, node:$a)]>;
+ [(AArch64and_u node:$pg, node:$a, node:$b),
+ (AArch64and_u node:$pg, node:$b, node:$a)]>;
def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2),
[(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2),
>From a43532f6fa965c1aa9c9cd0d9eafcde53317832a Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 23 Oct 2026 15:28:12 +0100
Subject: [PATCH 09/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Keep AND_U as an AArch64 target node until instruction selection. Remove the C++ combine and restore the TableGen patterns for BIC, register AND and immediate AND selection.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 34 -------------------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 31 +++++++++++++++++
2 files changed, 31 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 02a7bb7001194d..f806a3223ed4eb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25623,38 +25623,6 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
-static SDValue performSVEAndUCombine(SDNode *N, SelectionDAG &DAG) {
- SDValue Pg = N->getOperand(0);
- SDValue A = N->getOperand(1);
- SDValue B = N->getOperand(2);
-
- // Preserve matching predicates so TableGen can select:
- // and_u(pg, a, not(inactive, pg, b)) -> BIC(a, b)
- for (SDValue Not : {A, B}) {
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
- continue;
-
- SDValue NotPg = Not.getOperand(2);
- if (NotPg != Pg)
- continue;
-
- // We can discard AND_U's predicate if NOT's inactive lanes are
- // undefined or NOT's predicate is all-active: ordinary AND-NOT
- // patterns can select BIC.
- SDValue Inactive = Not.getOperand(1);
- if (Inactive.isUndef() || isAllActivePredicate(DAG, NotPg))
- continue;
-
- // Keep AND_U unchanged so TableGen can match the predicates.
- return SDValue();
- }
-
- // Otherwise expose an ordinary AND so generic DAG combines can simplify
- // it, for example:
- // and_u(pg, and_u(pg, a, 255), 15) -> and(a, 15)
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), A, B);
-}
-
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -32202,8 +32170,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performORCombine(N, DCI);
case ISD::AND:
return performANDCombine(N, DCI);
- case AArch64ISD::AND_U:
- return performSVEAndUCombine(N, DAG);
case ISD::FADD:
return performFADDCombine(N, DCI);
case AArch64ISD::FADD_PRED:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 099edf459391c9..889c5c4f7eee30 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -773,6 +773,26 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(VT (SVEAndU PT:$pg, VT:$a,
(VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
(BIC_ZZZ $a, $b)>;
+
+ // AND_U: NOT's undefined inactive lanes may take the value ~b,
+ // even when the predicates differ.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+
+ // AND_U with ordinary LLVM NOT, which produces ~b in every lane.
+ def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))),
+ (BIC_ZZZ $a, $b)>;
+
+ // Fall back to register AND when no BIC pattern matches.
+ // AND_U allows any result in inactive lanes.
+ def : Pat<(VT (AArch64and_u (PT srcvalue), VT:$a, VT:$b)),
+ (AND_ZZZ $a, $b)>;
}
defm : SVEAndNotPats<nxv16i8, nxv16i1>;
defm : SVEAndNotPats<nxv8i16, nxv8i1>;
@@ -837,6 +857,17 @@ let Predicates = [HasSVE_or_SME] in {
defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>;
defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>;
+ // Allow AND_U nodes to select AND-immediate instructions.
+ class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR,
+ ValueType IT, ComplexPattern CP>
+ : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a),
+ (VT (splat_vector (IT (CP i64:$imm)))))),
+ (AND_ZI_PSEUDO $a, i64:$imm)>;
+ def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>;
+ def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>;
+ def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>;
+ def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>;
+
defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>;
defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>;
defm UMAX_ZI : sve_int_arith_imm1_unsigned<0b01, "umax", AArch64umax_p>;
>From 6f119bf071f0b86fb93464f83fabe176ce0d1b20 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 23 Oct 2026 15:45:24 +0100
Subject: [PATCH 10/10] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _x forms
Check BIC selection for AND_U with poison NOT passthrough and different predicates. Cover the pattern independently of the matching-predicate case.
---
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 21 ++++---------------
1 file changed, 4 insertions(+), 17 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 4d8d595f242c24..13b4baa8c05549 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -124,23 +124,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4
; Poison NOT passthrough permits BIC even when the predicates differ.
define <vscale x 4 x i32> @different_predicates_poison(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg, <vscale x 4 x i1> %not_pg) {
-; SVE-LABEL: different_predicates_poison:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p1/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: different_predicates_poison:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p1/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: different_predicates_poison:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p1/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: different_predicates_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %not_pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
More information about the llvm-branch-commits
mailing list