[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms (PR #232526)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 16 01:02:33 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 8324795e551639fd503593b9048d97e536f650af Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 14 Oct 2026 23:22:09 +0100
Subject: [PATCH] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for
_x forms
Replace NOT followed by AND with BIC for expressions such as:
return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b));
Combine ordinary AND with NOT when the NOT predicate is all-active or
its inactive values are undefined.
Also combine matching-predicate SVE AND_x-NOT pairs before lowering
discards the AND predicate. AND_x ignores inactive results, so the NOT's
inactive values do not need preserving.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 36 ++++-
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++--------------
2 files changed, 56 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e3257af644550..bc3ded382f070 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22123,6 +22123,19 @@ static SDValue performReinterpretCastCombine(SDNode *N) {
static SDValue performSVEAndCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
SelectionDAG &DAG = DCI.DAG;
+
+ // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not
+ // matter, leaving standalone NOTs available for predicated selection.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Not = N->getOperand(I);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not)
+ continue;
+ if (Not.getOperand(1).isUndef() ||
+ isAllActivePredicate(DAG, Not.getOperand(2)))
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(1 - I), Not.getOperand(3));
+ }
+
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
@@ -25381,6 +25394,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64));
}
+static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ SDValue Pg = N->getOperand(1);
+
+ // Fold matching predicates before lowering AND_U discards its predicate.
+ // NOT's inactive lanes cannot affect AND_U's active lanes.
+ for (unsigned I = 0; I != 2; ++I) {
+ unsigned NotOperandIdx = 2 + I;
+ unsigned OtherOperandIdx = 3 - I;
+ SDValue Not = N->getOperand(NotOperandIdx);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
+ Not.getOperand(2) != Pg)
+ continue;
+
+ return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
+ N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ }
+ return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
+ N->getOperand(3));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25600,8 +25633,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
case Intrinsic::aarch64_sve_and_u:
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index cde00ee9f59d6..66ecfa31e01f2 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -6,26 +6,10 @@
; Ordinary LLVM AND with SVE NOT.
define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: and_not_ptrue_all:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_ptrue_all:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_ptrue_all:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_ptrue_all:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
@@ -33,49 +17,20 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_commuted:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z1.d, z0.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_commuted:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z1.d, z0.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_commuted:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z1.d, z0.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %not, %a
ret <vscale x 4 x i32> %and
}
define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: and_not_poison_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_poison_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_poison_i32:
-; SME: // %bb.0:
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_poison_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = and <vscale x 4 x i32> %a, %not
ret <vscale x 4 x i32> %and
@@ -85,26 +40,10 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4
; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate.
define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; SVE-LABEL: and_not_all_i32:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: and_not_all_i32:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: and_not_all_i32:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: and_not_all_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
@@ -113,8 +52,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_defined:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -124,8 +62,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: matching_commuted:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.d, z2.d, z0.d
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a)
@@ -133,26 +70,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4
}
define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: all_not_partial_and:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.d, z0.d, z1.d
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: all_not_partial_and:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.d, z0.d, z1.d
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: all_not_partial_and:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.d, z0.d, z1.d
-; SME-NEXT: ret
+; CHECK-LABEL: all_not_partial_and:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
More information about the llvm-branch-commits
mailing list