[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _m forms (PR #232527)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 22 08:52:25 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232527
>From 1e33a7216593e3897353cd4d4ebd8279bfe061c4 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 15 Oct 2026 22:23:45 +0100
Subject: [PATCH 1/2] [AArch64][llvm] Combine SVE AND-NOT operations into BIC
for _m forms
Replace NOT followed by merging AND with BIC for expressions such as:
return svand_u32_m(pg, a, svnot_u32_x(pg, b));
Combine when NOT is the second operand and its predicate matches the AND
predicate or is all-active.
Use predicated BIC to preserve the AND's first operand in inactive
lanes. Use unpredicated BIC when the AND predicate is all-active.
Do not commute merging AND operands, since this would change inactive
results. Leave pairs with incompatible predicates unchanged.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 33 ++++++++++
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 60 ++++---------------
2 files changed, 45 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 011c074992c99..44cd56e152a8a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25474,6 +25474,37 @@ static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
N->getOperand(3));
}
+static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and;
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ SDValue Pg = N->getOperand(1);
+
+ // Only commute AND_U: a merging AND must preserve its first operand in
+ // inactive lanes. NOT must cover all of AND's active lanes.
+ for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) {
+ unsigned NotOperandIdx = 3 - I;
+ unsigned OtherOperandIdx = 2 + I;
+ SDValue Not = N->getOperand(NotOperandIdx);
+ if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
+ (Not.getOperand(2) != Pg &&
+ !isAllActivePredicate(DAG, Not.getOperand(2))))
+ continue;
+
+ SDValue A = N->getOperand(OtherOperandIdx);
+ SDValue B = Not.getOperand(3);
+ if (!IsMerging || isAllActivePredicate(DAG, Pg))
+ return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B);
+
+ return DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B);
+ }
+ if (IsMerging)
+ return SDValue();
+ return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3));
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25692,6 +25723,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
N->getOperand(3));
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
+ case Intrinsic::aarch64_sve_and:
+ return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_and_u:
return performSVEAndUIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 1a1b53a90a746..ce12b583bfccd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -136,8 +136,7 @@ define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x
define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -147,8 +146,7 @@ define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.b, p0/m, z1.b
-; CHECK-NEXT: and z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: bic z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
%not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
%and = call <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %not)
@@ -158,8 +156,7 @@ define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 1
define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.h, p0/m, z1.h
-; CHECK-NEXT: and z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT: bic z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
%not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b)
%and = call <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %not)
@@ -169,8 +166,7 @@ define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x
define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.d, p0/m, z1.d
-; CHECK-NEXT: and z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b)
%and = call <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %not)
@@ -178,52 +174,20 @@ define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x
}
define <vscale x 4 x i32> @merging_and_all_active_not(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: merging_and_all_active_not:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: not z1.s, p1/m, z1.s
-; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: merging_and_all_active_not:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p1.s
-; SVE2P2-NEXT: not z1.s, p1/z, z1.s
-; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: merging_and_all_active_not:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p1.s
-; SME-NEXT: not z1.s, p1/m, z1.s
-; SME-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SME-NEXT: ret
+; CHECK-LABEL: merging_and_all_active_not:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
}
define <vscale x 4 x i32> @merging_and_all_active(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: merging_and_all_active:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: merging_and_all_active:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: merging_and_all_active:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SME-NEXT: ret
+; CHECK-LABEL: merging_and_all_active:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
>From 7f8590cc2d099aef6138b566d495c0c01c5eb37e Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Wed, 21 Oct 2026 16:32:26 +0100
Subject: [PATCH 2/2] fixup! [AArch64][llvm] Combine SVE AND-NOT operations
into BIC for _m forms
Select merging AND-NOT operations with TableGen patterns instead of the C++ combine. Preserve the first operand in inactive lanes and use an unpredicated BIC when both predicates are all-active.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 33 -------------------
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 20 +++++++++++
2 files changed, 20 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 44cd56e152a8a..011c074992c99 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25474,37 +25474,6 @@ static SDValue performSVEAndUIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
N->getOperand(3));
}
-static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
- bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and;
- SDLoc DL(N);
- EVT VT = N->getValueType(0);
- SDValue Pg = N->getOperand(1);
-
- // Only commute AND_U: a merging AND must preserve its first operand in
- // inactive lanes. NOT must cover all of AND's active lanes.
- for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) {
- unsigned NotOperandIdx = 3 - I;
- unsigned OtherOperandIdx = 2 + I;
- SDValue Not = N->getOperand(NotOperandIdx);
- if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
- (Not.getOperand(2) != Pg &&
- !isAllActivePredicate(DAG, Not.getOperand(2))))
- continue;
-
- SDValue A = N->getOperand(OtherOperandIdx);
- SDValue B = Not.getOperand(3);
- if (!IsMerging || isAllActivePredicate(DAG, Pg))
- return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B);
-
- return DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, VT,
- DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B);
- }
- if (IsMerging)
- return SDValue();
- return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3));
-}
-
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -25723,8 +25692,6 @@ static SDValue performIntrinsicCombine(SDNode *N,
N->getOperand(3));
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
- case Intrinsic::aarch64_sve_and:
- return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_and_u:
return performSVEAndUIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 8ffdb8cb9c05f..a1869a199dd31 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -786,6 +786,26 @@ let Predicates = [HasSVE_or_SME] in {
defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>;
defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>;
defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>;
+
+ multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT,
+ Instruction PredBic> {
+ // Preserve A in inactive lanes. NOT must cover every active AND lane.
+ def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))),
+ (PredBic $pg, $a, $b)>;
+ def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (PredBic $pg, $a, $b)>;
+
+ // With both predicates all-active, no inactive lanes need preserving.
+ def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a,
+ (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))),
+ (BIC_ZZZ $a, $b)>;
+ }
+ defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>;
+ defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>;
+ defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>;
+ defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>;
} // End HasSVE_or_SME
let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in {
More information about the llvm-branch-commits
mailing list