[llvm-branch-commits] [llvm] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _m forms (PR #232527)
Jonathan Thackray via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 16 01:02:33 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232527
>From 5c4cf2ff515f5a098f4a9fa27c0598f0ebabed23 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 15 Oct 2026 22:23:45 +0100
Subject: [PATCH] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for
_m forms
Replace NOT followed by merging AND with BIC for expressions such as:
return svand_u32_m(pg, a, svnot_u32_x(pg, b));
Combine when NOT is the second operand and its predicate matches the AND
predicate or is all-active.
Use predicated BIC to preserve the AND's first operand in inactive
lanes. Use unpredicated BIC when the AND predicate is all-active.
Do not commute merging AND operands, since this would change inactive
results. Leave pairs with incompatible predicates unchanged.
This applies to all integer element widths. Update the pre-commit LLVM
expectations, including cases where combining would change the
result.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 32 ++++++----
.../CodeGen/AArch64/sve-intrinsics-and-not.ll | 60 ++++---------------
2 files changed, 34 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bc3ded382f070..9859924fc0736 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25395,23 +25395,34 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N,
}
static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
+ bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and;
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
SDValue Pg = N->getOperand(1);
- // Fold matching predicates before lowering AND_U discards its predicate.
- // NOT's inactive lanes cannot affect AND_U's active lanes.
- for (unsigned I = 0; I != 2; ++I) {
- unsigned NotOperandIdx = 2 + I;
- unsigned OtherOperandIdx = 3 - I;
+ // Only commute AND_U: a merging AND must preserve its first operand in
+ // inactive lanes. NOT must cover all of AND's active lanes.
+ for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) {
+ unsigned NotOperandIdx = 3 - I;
+ unsigned OtherOperandIdx = 2 + I;
SDValue Not = N->getOperand(NotOperandIdx);
if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not ||
- Not.getOperand(2) != Pg)
+ (Not.getOperand(2) != Pg &&
+ !isAllActivePredicate(DAG, Not.getOperand(2))))
continue;
- return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
- N->getOperand(OtherOperandIdx), Not.getOperand(3));
+ SDValue A = N->getOperand(OtherOperandIdx);
+ SDValue B = Not.getOperand(3);
+ if (!IsMerging || isAllActivePredicate(DAG, Pg))
+ return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B);
+
+ return DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, VT,
+ DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B);
}
- return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2),
- N->getOperand(3));
+ if (IsMerging)
+ return SDValue();
+ return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3));
}
static SDValue performIntrinsicCombine(SDNode *N,
@@ -25632,6 +25643,7 @@ static SDValue performIntrinsicCombine(SDNode *N,
N->getOperand(3));
case Intrinsic::aarch64_sve_subr:
return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true);
+ case Intrinsic::aarch64_sve_and:
case Intrinsic::aarch64_sve_and_u:
return performSVEAndNotIntrinsicCombine(N, DAG);
case Intrinsic::aarch64_sve_bic_u:
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
index 66ecfa31e01f2..c988622be8407 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll
@@ -84,8 +84,7 @@ define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x
define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.s, p0/m, z1.s
-; CHECK-NEXT: and z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
@@ -95,8 +94,7 @@ define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x
define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.b, p0/m, z1.b
-; CHECK-NEXT: and z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: bic z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
%not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b)
%and = call <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %not)
@@ -106,8 +104,7 @@ define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 1
define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.h, p0/m, z1.h
-; CHECK-NEXT: and z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT: bic z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
%not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b)
%and = call <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %not)
@@ -117,8 +114,7 @@ define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x
define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg) {
; CHECK-LABEL: merging_and_rhs_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: not z2.d, p0/m, z1.d
-; CHECK-NEXT: and z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: bic z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b)
%and = call <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %not)
@@ -126,52 +122,20 @@ define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x
}
define <vscale x 4 x i32> @merging_and_all_active_not(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) {
-; SVE-LABEL: merging_and_all_active_not:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: not z1.s, p1/m, z1.s
-; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: merging_and_all_active_not:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p1.s
-; SVE2P2-NEXT: not z1.s, p1/z, z1.s
-; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: merging_and_all_active_not:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p1.s
-; SME-NEXT: not z1.s, p1/m, z1.s
-; SME-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SME-NEXT: ret
+; CHECK-LABEL: merging_and_all_active_not:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
}
define <vscale x 4 x i32> @merging_and_all_active(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) {
-; SVE-LABEL: merging_and_all_active:
-; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: not z1.s, p0/m, z1.s
-; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE-NEXT: ret
-;
-; SVE2P2-LABEL: merging_and_all_active:
-; SVE2P2: // %bb.0:
-; SVE2P2-NEXT: ptrue p0.s
-; SVE2P2-NEXT: not z1.s, p0/z, z1.s
-; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SVE2P2-NEXT: ret
-;
-; SME-LABEL: merging_and_all_active:
-; SME: // %bb.0:
-; SME-NEXT: ptrue p0.s
-; SME-NEXT: not z1.s, p0/m, z1.s
-; SME-NEXT: and z0.s, p0/m, z0.s, z1.s
-; SME-NEXT: ret
+; CHECK-LABEL: merging_and_all_active:
+; CHECK: // %bb.0:
+; CHECK-NEXT: bic z0.d, z0.d, z1.d
+; CHECK-NEXT: ret
%not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b)
%and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not)
ret <vscale x 4 x i32> %and
More information about the llvm-branch-commits
mailing list