[llvm] [AArch64] Match BSP through trunc(and) canonicalization in OR combine (PR #198277)
Mariusz Masztalerczuk via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 23:53:48 PDT 2026
https://github.com/mmasztalerczuk updated https://github.com/llvm/llvm-project/pull/198277
>From b09150ef8eac508dad38f7f9143d33f78ff37a71 Mon Sep 17 00:00:00 2001
From: Mariusz Masztalerczuk <mariusz at masztalerczuk.com>
Date: Mon, 18 May 2026 09:26:09 +0200
Subject: [PATCH 1/5] [AArch64] Match BSP through trunc(and) canonicalization
in OR combine
When lowering `v[N]i1 select + mask + sext` patterns (e.g. the canonical
`select(cmp_a, gt_lanes, lt_lanes)` masked by another setcc), the OR that
implements the BSP-equivalent ends up after DAGCombiner has canonicalized
(and (trunc x) (trunc y)) -> (trunc (and x y))
so the post-legalize OR has shape
(or (trunc (and A B)) (and C (xor (trunc A) -1)))
That shape is not recognized by `tryLowerToBSL` (called early from
`LowerVectorOR`, before the canonicalization) nor by the BSP TableGen
pattern, so AArch64 emits a redundant `bic + orr + and` sequence instead
of a single `BIF/BSL/BIT`.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 57 ++++
.../CodeGen/AArch64/bsp-vselect-i1-sext.ll | 262 ++++++++++++++++++
2 files changed, 319 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 413b585503335..ed96091a2e9d7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -21078,6 +21078,60 @@ static SDValue performANDORDUPNOTCombine(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(Opc, DL, VT, X, Not);
}
+/// Fold (or (trunc (and A B)) (and C (xor (trunc A) -1))) into
+/// BSP(trunc(A), trunc(B), C). DAGCombiner canonicalizes
+/// (and (trunc x) (trunc y)) into (trunc (and x y)), which hides this BSP
+/// shape from both tryLowerToBSL and the BSP TableGen pattern; rewrite it
+/// here so ISel emits BIF/BSL/BIT.
+static SDValue performORBSLTruncCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG) {
+ EVT VT = N->getValueType(0);
+ if (DCI.isBeforeLegalizeOps() || !VT.isFixedLengthVector() ||
+ !DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() ||
+ // BSP has TableGen patterns only for legal NEON vector types; emitting
+ // it in an illegal type (e.g. v4i8) crashes the type legalizer.
+ !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+ return SDValue();
+
+ for (unsigned Side = 0; Side != 2; ++Side) {
+ // Side = which operand of OR holds (trunc (and A B)).
+ SDValue TruncSide = N->getOperand(Side);
+ SDValue AndSide = N->getOperand(1 - Side);
+ if (TruncSide.getOpcode() != ISD::TRUNCATE || !TruncSide.hasOneUse() ||
+ AndSide.getOpcode() != ISD::AND)
+ continue;
+ SDValue InnerAnd = TruncSide.getOperand(0);
+ if (InnerAnd.getOpcode() != ISD::AND || !InnerAnd.hasOneUse())
+ continue;
+
+ // Find the NOT-of-trunc and the other operand of the second AND.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue NotOp = AndSide.getOperand(I);
+ if (!isBitwiseNot(NotOp))
+ continue;
+ SDValue MaskTrunc = NotOp.getOperand(0);
+ if (MaskTrunc.getOpcode() != ISD::TRUNCATE)
+ continue;
+ SDValue WideA = MaskTrunc.getOperand(0);
+ SDValue WideB;
+ if (InnerAnd.getOperand(0) == WideA)
+ WideB = InnerAnd.getOperand(1);
+ else if (InnerAnd.getOperand(1) == WideA)
+ WideB = InnerAnd.getOperand(0);
+ else
+ continue;
+
+ SDLoc DL(N);
+ return DAG.getNode(AArch64ISD::BSP, DL, VT,
+ DAG.getNode(ISD::TRUNCATE, DL, VT, WideA),
+ DAG.getNode(ISD::TRUNCATE, DL, VT, WideB),
+ AndSide.getOperand(1 - I));
+ }
+ }
+ return SDValue();
+}
+
static SDValue performORCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
SelectionDAG &DAG = DCI.DAG;
@@ -21091,6 +21145,9 @@ static SDValue performORCombine(SDNode *N,
if (SDValue R = performANDORDUPNOTCombine(N, DAG))
return R;
+ if (SDValue R = performORBSLTruncCombine(N, DCI, DAG))
+ return R;
+
return SDValue();
}
diff --git a/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll b/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
new file mode 100644
index 0000000000000..4aed4c1d6b392
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+neon -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-unknown-linux-gnu -mattr=+neon -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+;; A v4i1 select + mask + sext on f32 inputs. After type legalization the
+;; inner OR has shape (or (trunc (and a b)) (and (trunc c) (xor (trunc a) -1)))
+;; which is bitwise-equivalent to a BSP. Should fold to a single BIF/BSL/BIT.
+
+define <4 x i32> @bsp_v4i1_select_mask_sext(<4 x float> %a, <4 x float> %b) {
+; CHECK-LE-LABEL: bsp_v4i1_select_mask_sext:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: fcmeq v3.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmgt v4.4s, v1.4s, v0.4s
+; CHECK-LE-NEXT: cmgt v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmge v1.4s, v2.4s, #0
+; CHECK-LE-NEXT: xtn v2.4h, v4.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: mvn v3.16b, v3.16b
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: xtn v3.4h, v3.4s
+; CHECK-LE-NEXT: bif v0.8b, v2.8b, v1.8b
+; CHECK-LE-NEXT: and v0.8b, v3.8b, v0.8b
+; CHECK-LE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v4i1_select_mask_sext:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: fcmeq v3.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmgt v4.4s, v1.4s, v0.4s
+; CHECK-BE-NEXT: cmgt v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmge v1.4s, v2.4s, #0
+; CHECK-BE-NEXT: xtn v2.4h, v4.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: mvn v3.16b, v3.16b
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: xtn v3.4h, v3.4s
+; CHECK-BE-NEXT: bif v0.8b, v2.8b, v1.8b
+; CHECK-BE-NEXT: and v0.8b, v3.8b, v0.8b
+; CHECK-BE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+entry:
+ %ai = bitcast <4 x float> %a to <4 x i32>
+ %bi = bitcast <4 x float> %b to <4 x i32>
+ %une = fcmp une <4 x float> %a, %b
+ %lt = icmp slt <4 x i32> %ai, %bi
+ %gt = icmp sgt <4 x i32> %ai, %bi
+ %and = and <4 x i32> %ai, %bi
+ %sign = icmp sge <4 x i32> %and, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i1> %gt, <4 x i1> %lt
+ %mask = and <4 x i1> %une, %sel
+ %result = sext <4 x i1> %mask to <4 x i32>
+ ret <4 x i32> %result
+}
+
+;; Same shape, but operands of the inner OR swapped in IR. The combine must
+;; be commutative in OR's operands.
+
+define <4 x i32> @bsp_v4i1_select_mask_sext_swapped(<4 x float> %a, <4 x float> %b) {
+; CHECK-LE-LABEL: bsp_v4i1_select_mask_sext_swapped:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: fcmeq v3.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmgt v4.4s, v1.4s, v0.4s
+; CHECK-LE-NEXT: cmgt v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmge v1.4s, v2.4s, #0
+; CHECK-LE-NEXT: xtn v2.4h, v4.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: mvn v3.16b, v3.16b
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: xtn v3.4h, v3.4s
+; CHECK-LE-NEXT: bit v0.8b, v2.8b, v1.8b
+; CHECK-LE-NEXT: and v0.8b, v3.8b, v0.8b
+; CHECK-LE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v4i1_select_mask_sext_swapped:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: fcmeq v3.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmgt v4.4s, v1.4s, v0.4s
+; CHECK-BE-NEXT: cmgt v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmge v1.4s, v2.4s, #0
+; CHECK-BE-NEXT: xtn v2.4h, v4.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: mvn v3.16b, v3.16b
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: xtn v3.4h, v3.4s
+; CHECK-BE-NEXT: bit v0.8b, v2.8b, v1.8b
+; CHECK-BE-NEXT: and v0.8b, v3.8b, v0.8b
+; CHECK-BE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+entry:
+ %ai = bitcast <4 x float> %a to <4 x i32>
+ %bi = bitcast <4 x float> %b to <4 x i32>
+ %une = fcmp une <4 x float> %a, %b
+ %lt = icmp slt <4 x i32> %ai, %bi
+ %gt = icmp sgt <4 x i32> %ai, %bi
+ %and = and <4 x i32> %ai, %bi
+ %sign = icmp sge <4 x i32> %and, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i1> %lt, <4 x i1> %gt
+ %mask = and <4 x i1> %une, %sel
+ %result = sext <4 x i1> %mask to <4 x i32>
+ ret <4 x i32> %result
+}
+
+;; Select-only variant (no outer fcmp+and+sext). Combine should fire on the
+;; OR directly and emit BSP in the wider type v4i32.
+
+define <4 x i16> @bsp_v4i1_select_only(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LE-LABEL: bsp_v4i1_select_only:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: bsl v0.16b, v1.16b, v2.16b
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v4i1_select_only:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: bsl v0.16b, v1.16b, v2.16b
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+entry:
+ %sign = icmp sge <4 x i32> %a, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i32> %b, <4 x i32> %c
+ %t = trunc <4 x i32> %sel to <4 x i16>
+ ret <4 x i16> %t
+}
+
+;; Wider lane (v8i16 mask coming from v8i32 setcc). Verifies the combine is
+;; not type-specific to v4i32.
+
+define <8 x i16> @bsp_v8i1_select_wide(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) {
+; CHECK-LE-LABEL: bsp_v8i1_select_wide:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: cmge v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: bsl v1.16b, v3.16b, v5.16b
+; CHECK-LE-NEXT: bsl v0.16b, v2.16b, v4.16b
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v8i1_select_wide:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: rev64 v5.4s, v5.4s
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v5.16b, v5.16b, v5.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmge v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: bsl v1.16b, v3.16b, v5.16b
+; CHECK-BE-NEXT: bsl v0.16b, v2.16b, v4.16b
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+entry:
+ %sign = icmp sge <8 x i32> %a, zeroinitializer
+ %sel = select <8 x i1> %sign, <8 x i32> %b, <8 x i32> %c
+ %t = trunc <8 x i32> %sel to <8 x i16>
+ ret <8 x i16> %t
+}
+
+;; Negative control: when the mask is a true splat constant, the existing
+;; constant-mask path in tryLowerToBSL should win; the new combine must not
+;; interfere.
+
+define <4 x i16> @bsp_const_mask(<4 x i16> %b, <4 x i16> %c) {
+; CHECK-LE-LABEL: bsp_const_mask:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: movi d2, #0x00ffff0000ffff
+; CHECK-LE-NEXT: bif v0.8b, v1.8b, v2.8b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_const_mask:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: movi d2, #0x00ffff0000ffff
+; CHECK-BE-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: bif v0.8b, v1.8b, v2.8b
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+entry:
+ %m = and <4 x i16> %b, <i16 -1, i16 0, i16 -1, i16 0>
+ %n = and <4 x i16> %c, <i16 0, i16 -1, i16 0, i16 -1>
+ %r = or <4 x i16> %m, %n
+ ret <4 x i16> %r
+}
+
+;; Negative control: the inner `and(gt, sign)` in the matched pattern has a
+;; second user (returned alongside the masked OR). The combine must bail
+;; out via `InnerAnd.hasOneUse()` so the wider AND is not duplicated.
+
+define { <4 x i16>, <4 x i32> } @bsp_no_fold_multi_use(<4 x i32> %gt, <4 x i32> %sign, <4 x i16> %lt_n) {
+; CHECK-LE-LABEL: bsp_no_fold_multi_use:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: and v3.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: xtn v0.4h, v1.4s
+; CHECK-LE-NEXT: xtn v1.4h, v3.4s
+; CHECK-LE-NEXT: bic v0.8b, v2.8b, v0.8b
+; CHECK-LE-NEXT: orr v0.8b, v1.8b, v0.8b
+; CHECK-LE-NEXT: mov v1.16b, v3.16b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_no_fold_multi_use:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: xtn v3.4h, v0.4s
+; CHECK-BE-NEXT: bic v1.8b, v2.8b, v1.8b
+; CHECK-BE-NEXT: rev64 v2.4s, v0.4s
+; CHECK-BE-NEXT: orr v0.8b, v3.8b, v1.8b
+; CHECK-BE-NEXT: ext v1.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+entry:
+ %wand = and <4 x i32> %gt, %sign
+ %tand = trunc <4 x i32> %wand to <4 x i16>
+ %tsign = trunc <4 x i32> %sign to <4 x i16>
+ %nsign = xor <4 x i16> %tsign, splat (i16 -1)
+ %rhs = and <4 x i16> %lt_n, %nsign
+ %or = or <4 x i16> %tand, %rhs
+ %r0 = insertvalue { <4 x i16>, <4 x i32> } poison, <4 x i16> %or, 0
+ %r1 = insertvalue { <4 x i16>, <4 x i32> } %r0, <4 x i32> %wand, 1
+ ret { <4 x i16>, <4 x i32> } %r1
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From 4b68663ea56a803b68c5a40755674872cd083ad4 Mon Sep 17 00:00:00 2001
From: Mariusz Masztalerczuk <mariusz at masztalerczuk.com>
Date: Thu, 28 May 2026 04:41:37 +0200
Subject: [PATCH 2/5] [AArch64] Consolidate trunc(and) BSP matching into
tryLowerToBSL
Move the matcher into tryLowerToBSL and additionally invoke
tryLowerToBSL from performORCombine post-legalize-ops, where the
canonicalized (or (trunc (and A B)) (and C (xor (trunc A) -1))) shape
becomes visible. An isTypeLegal(VT) guard at the top of tryLowerToBSL
covers both call sites. performORBSLTruncCombine is removed.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 102 ++++++++----------
1 file changed, 46 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ed96091a2e9d7..baf1f62c64819 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15804,6 +15804,47 @@ static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
if (VT.isScalableVector() && !Subtarget.hasSVE2())
return SDValue();
+ // BSP has TableGen patterns only for legal NEON vector types.
+ if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
+ return SDValue();
+
+ // Match the trunc-hoisted shape that hides BSP from the cases below:
+ // (or (trunc (and A B)) (and C (xor (trunc A) -1))) => BSP(trunc A, trunc B, C)
+ if (VT.isFixedLengthVector() && Subtarget.isNeonAvailable()) {
+ for (unsigned Side = 0; Side != 2; ++Side) {
+ SDValue TruncSide = N->getOperand(Side);
+ SDValue AndSide = N->getOperand(1 - Side);
+ if (TruncSide.getOpcode() != ISD::TRUNCATE || !TruncSide.hasOneUse() ||
+ AndSide.getOpcode() != ISD::AND)
+ continue;
+ SDValue InnerAnd = TruncSide.getOperand(0);
+ if (InnerAnd.getOpcode() != ISD::AND || !InnerAnd.hasOneUse())
+ continue;
+
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue NotOp = AndSide.getOperand(I);
+ if (!isBitwiseNot(NotOp))
+ continue;
+ SDValue MaskTrunc = NotOp.getOperand(0);
+ if (MaskTrunc.getOpcode() != ISD::TRUNCATE)
+ continue;
+ SDValue WideA = MaskTrunc.getOperand(0);
+ SDValue WideB;
+ if (InnerAnd.getOperand(0) == WideA)
+ WideB = InnerAnd.getOperand(1);
+ else if (InnerAnd.getOperand(1) == WideA)
+ WideB = InnerAnd.getOperand(0);
+ else
+ continue;
+
+ return DAG.getNode(AArch64ISD::BSP, DL, VT,
+ DAG.getNode(ISD::TRUNCATE, DL, VT, WideA),
+ DAG.getNode(ISD::TRUNCATE, DL, VT, WideB),
+ AndSide.getOperand(1 - I));
+ }
+ }
+ }
+
SDValue N0 = N->getOperand(0);
if (N0.getOpcode() != ISD::AND)
return SDValue();
@@ -21078,60 +21119,6 @@ static SDValue performANDORDUPNOTCombine(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(Opc, DL, VT, X, Not);
}
-/// Fold (or (trunc (and A B)) (and C (xor (trunc A) -1))) into
-/// BSP(trunc(A), trunc(B), C). DAGCombiner canonicalizes
-/// (and (trunc x) (trunc y)) into (trunc (and x y)), which hides this BSP
-/// shape from both tryLowerToBSL and the BSP TableGen pattern; rewrite it
-/// here so ISel emits BIF/BSL/BIT.
-static SDValue performORBSLTruncCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI,
- SelectionDAG &DAG) {
- EVT VT = N->getValueType(0);
- if (DCI.isBeforeLegalizeOps() || !VT.isFixedLengthVector() ||
- !DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() ||
- // BSP has TableGen patterns only for legal NEON vector types; emitting
- // it in an illegal type (e.g. v4i8) crashes the type legalizer.
- !DAG.getTargetLoweringInfo().isTypeLegal(VT))
- return SDValue();
-
- for (unsigned Side = 0; Side != 2; ++Side) {
- // Side = which operand of OR holds (trunc (and A B)).
- SDValue TruncSide = N->getOperand(Side);
- SDValue AndSide = N->getOperand(1 - Side);
- if (TruncSide.getOpcode() != ISD::TRUNCATE || !TruncSide.hasOneUse() ||
- AndSide.getOpcode() != ISD::AND)
- continue;
- SDValue InnerAnd = TruncSide.getOperand(0);
- if (InnerAnd.getOpcode() != ISD::AND || !InnerAnd.hasOneUse())
- continue;
-
- // Find the NOT-of-trunc and the other operand of the second AND.
- for (unsigned I = 0; I != 2; ++I) {
- SDValue NotOp = AndSide.getOperand(I);
- if (!isBitwiseNot(NotOp))
- continue;
- SDValue MaskTrunc = NotOp.getOperand(0);
- if (MaskTrunc.getOpcode() != ISD::TRUNCATE)
- continue;
- SDValue WideA = MaskTrunc.getOperand(0);
- SDValue WideB;
- if (InnerAnd.getOperand(0) == WideA)
- WideB = InnerAnd.getOperand(1);
- else if (InnerAnd.getOperand(1) == WideA)
- WideB = InnerAnd.getOperand(0);
- else
- continue;
-
- SDLoc DL(N);
- return DAG.getNode(AArch64ISD::BSP, DL, VT,
- DAG.getNode(ISD::TRUNCATE, DL, VT, WideA),
- DAG.getNode(ISD::TRUNCATE, DL, VT, WideB),
- AndSide.getOperand(1 - I));
- }
- }
- return SDValue();
-}
-
static SDValue performORCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
SelectionDAG &DAG = DCI.DAG;
@@ -21145,8 +21132,11 @@ static SDValue performORCombine(SDNode *N,
if (SDValue R = performANDORDUPNOTCombine(N, DAG))
return R;
- if (SDValue R = performORBSLTruncCombine(N, DCI, DAG))
- return R;
+ // Re-try BSL post-legalize: the trunc-hoisted BSP shape only appears after
+ // LowerVectorOR (and its early tryLowerToBSL call) has run.
+ if (!DCI.isBeforeLegalizeOps() && N->getValueType(0).isVector())
+ if (SDValue R = tryLowerToBSL(SDValue(N, 0), DAG))
+ return R;
return SDValue();
}
>From 51a210a11c6bf8fe33999093d5a7a867c05a79ef Mon Sep 17 00:00:00 2001
From: Mariusz Masztalerczuk <mariusz at masztalerczuk.com>
Date: Thu, 28 May 2026 09:23:06 +0200
Subject: [PATCH 3/5] [AArch64] Use SDPatternMatch for trunc(and) BSP matcher
Address review feedback: replace the chain of opcode if-checks with
sd_match. The outer commutativity of the OR (and the inner AND of
A/B inside the trunc) is now handled by the matcher itself, so the
outer Side loop is gone. m_Not is intentionally not used because it
relies on the strict m_AllOnes (no AllowTruncation), which misses the
post-legalize NOT shape produced for fixed-length vectors; isBitwiseNot
is kept for that check, with a small two-step Swap loop to disambiguate
between the NOT-of-trunc operand and the bystander C.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 40 ++++++-------------
1 file changed, 13 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index baf1f62c64819..dd0ec323b88c5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15811,36 +15811,22 @@ static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
// Match the trunc-hoisted shape that hides BSP from the cases below:
// (or (trunc (and A B)) (and C (xor (trunc A) -1))) => BSP(trunc A, trunc B, C)
if (VT.isFixedLengthVector() && Subtarget.isNeonAvailable()) {
- for (unsigned Side = 0; Side != 2; ++Side) {
- SDValue TruncSide = N->getOperand(Side);
- SDValue AndSide = N->getOperand(1 - Side);
- if (TruncSide.getOpcode() != ISD::TRUNCATE || !TruncSide.hasOneUse() ||
- AndSide.getOpcode() != ISD::AND)
- continue;
- SDValue InnerAnd = TruncSide.getOperand(0);
- if (InnerAnd.getOpcode() != ISD::AND || !InnerAnd.hasOneUse())
- continue;
-
- for (unsigned I = 0; I != 2; ++I) {
- SDValue NotOp = AndSide.getOperand(I);
- if (!isBitwiseNot(NotOp))
+ using namespace SDPatternMatch;
+ SDValue InnerLHS, InnerRHS, NotOp, C;
+ if (sd_match(N, m_Or(m_OneUse(m_Trunc(m_OneUse(m_And(
+ m_Value(InnerLHS), m_Value(InnerRHS))))),
+ m_And(m_Value(NotOp), m_Value(C))))) {
+ for (unsigned Swap = 0; Swap != 2; ++Swap, std::swap(NotOp, C)) {
+ if (!isBitwiseNot(NotOp) ||
+ NotOp.getOperand(0).getOpcode() != ISD::TRUNCATE)
continue;
- SDValue MaskTrunc = NotOp.getOperand(0);
- if (MaskTrunc.getOpcode() != ISD::TRUNCATE)
+ SDValue MaskWide = NotOp.getOperand(0).getOperand(0);
+ if (MaskWide != InnerLHS && MaskWide != InnerRHS)
continue;
- SDValue WideA = MaskTrunc.getOperand(0);
- SDValue WideB;
- if (InnerAnd.getOperand(0) == WideA)
- WideB = InnerAnd.getOperand(1);
- else if (InnerAnd.getOperand(1) == WideA)
- WideB = InnerAnd.getOperand(0);
- else
- continue;
-
+ SDValue WideB = MaskWide == InnerLHS ? InnerRHS : InnerLHS;
return DAG.getNode(AArch64ISD::BSP, DL, VT,
- DAG.getNode(ISD::TRUNCATE, DL, VT, WideA),
- DAG.getNode(ISD::TRUNCATE, DL, VT, WideB),
- AndSide.getOperand(1 - I));
+ DAG.getNode(ISD::TRUNCATE, DL, VT, MaskWide),
+ DAG.getNode(ISD::TRUNCATE, DL, VT, WideB), C);
}
}
}
>From c63f0a3f31b8bf479ed826e5ea36ba5ca1751b8f Mon Sep 17 00:00:00 2001
From: Mariusz Masztalerczuk <mariusz at masztalerczuk.com>
Date: Tue, 2 Jun 2026 16:16:43 +0200
Subject: [PATCH 4/5] [AArch64] Use BitwiseNotOfTruncMatch helper in
tryLowerToBSL
---
.../Target/AArch64/AArch64ISelLowering.cpp | 32 ++++++++++++-------
1 file changed, 20 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dd0ec323b88c5..54cb7637a7a25 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15795,6 +15795,18 @@ static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(Inst, DL, VT, X, Y, Imm);
}
+// Matches (xor (trunc x), -1) and binds x; uses isBitwiseNot so post-legalize
+// splats with wider element storage than the lane width still match.
+struct BitwiseNotOfTruncMatch {
+ SDValue &Wide;
+ template <typename Ctx> bool match(const Ctx &, SDValue N) const {
+ if (!isBitwiseNot(N) || N.getOperand(0).getOpcode() != ISD::TRUNCATE)
+ return false;
+ Wide = N.getOperand(0).getOperand(0);
+ return true;
+ }
+};
+
static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
assert(VT.isVector() && "Expected vector type in tryLowerToBSL\n");
@@ -15809,20 +15821,16 @@ static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
return SDValue();
// Match the trunc-hoisted shape that hides BSP from the cases below:
- // (or (trunc (and A B)) (and C (xor (trunc A) -1))) => BSP(trunc A, trunc B, C)
+ // (or (trunc (and A B)) (and C (xor (trunc A) -1)))
+ // => BSP(trunc A, trunc B, C)
if (VT.isFixedLengthVector() && Subtarget.isNeonAvailable()) {
using namespace SDPatternMatch;
- SDValue InnerLHS, InnerRHS, NotOp, C;
- if (sd_match(N, m_Or(m_OneUse(m_Trunc(m_OneUse(m_And(
- m_Value(InnerLHS), m_Value(InnerRHS))))),
- m_And(m_Value(NotOp), m_Value(C))))) {
- for (unsigned Swap = 0; Swap != 2; ++Swap, std::swap(NotOp, C)) {
- if (!isBitwiseNot(NotOp) ||
- NotOp.getOperand(0).getOpcode() != ISD::TRUNCATE)
- continue;
- SDValue MaskWide = NotOp.getOperand(0).getOperand(0);
- if (MaskWide != InnerLHS && MaskWide != InnerRHS)
- continue;
+ SDValue InnerLHS, InnerRHS, MaskWide, C;
+ if (sd_match(
+ N, m_Or(m_OneUse(m_Trunc(
+ m_OneUse(m_And(m_Value(InnerLHS), m_Value(InnerRHS))))),
+ m_And(BitwiseNotOfTruncMatch{MaskWide}, m_Value(C))))) {
+ if (MaskWide == InnerLHS || MaskWide == InnerRHS) {
SDValue WideB = MaskWide == InnerLHS ? InnerRHS : InnerLHS;
return DAG.getNode(AArch64ISD::BSP, DL, VT,
DAG.getNode(ISD::TRUNCATE, DL, VT, MaskWide),
>From e55fa8e546fb88d9ef2cf8280d77d566178919d0 Mon Sep 17 00:00:00 2001
From: Mariusz Masztalerczuk <mariusz at masztalerczuk.com>
Date: Wed, 10 Jun 2026 04:24:58 +0200
Subject: [PATCH 5/5] [AArch64] Use m_ConstInt for trunc-xor BSP match; add
commutative tests
Replace the custom BitwiseNotOfTruncMatch helper with stock matchers:
m_ConstInt
Add direct-shape tests that flip each commutative operand order and a negative non-BSP case.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 28 +-
.../CodeGen/AArch64/bsp-vselect-i1-sext.ll | 258 ++++++++++++++++++
2 files changed, 269 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 54cb7637a7a25..c562a8ad22473 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15795,18 +15795,6 @@ static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(Inst, DL, VT, X, Y, Imm);
}
-// Matches (xor (trunc x), -1) and binds x; uses isBitwiseNot so post-legalize
-// splats with wider element storage than the lane width still match.
-struct BitwiseNotOfTruncMatch {
- SDValue &Wide;
- template <typename Ctx> bool match(const Ctx &, SDValue N) const {
- if (!isBitwiseNot(N) || N.getOperand(0).getOpcode() != ISD::TRUNCATE)
- return false;
- Wide = N.getOperand(0).getOperand(0);
- return true;
- }
-};
-
static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
assert(VT.isVector() && "Expected vector type in tryLowerToBSL\n");
@@ -15823,14 +15811,20 @@ static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
// Match the trunc-hoisted shape that hides BSP from the cases below:
// (or (trunc (and A B)) (and C (xor (trunc A) -1)))
// => BSP(trunc A, trunc B, C)
+ // m_Or, m_And and m_Xor match operands in any order. The xor constant uses
+ // m_ConstInt + isAllOnes() rather than m_AllOnes because m_AllOnes rejects
+ // post-legalize splats whose element storage is wider than the lane.
if (VT.isFixedLengthVector() && Subtarget.isNeonAvailable()) {
using namespace SDPatternMatch;
SDValue InnerLHS, InnerRHS, MaskWide, C;
- if (sd_match(
- N, m_Or(m_OneUse(m_Trunc(
- m_OneUse(m_And(m_Value(InnerLHS), m_Value(InnerRHS))))),
- m_And(BitwiseNotOfTruncMatch{MaskWide}, m_Value(C))))) {
- if (MaskWide == InnerLHS || MaskWide == InnerRHS) {
+ APInt NotMask;
+ if (sd_match(N, m_Or(m_OneUse(m_Trunc(m_OneUse(
+ m_And(m_Value(InnerLHS), m_Value(InnerRHS))))),
+ m_And(m_Xor(m_Trunc(m_Value(MaskWide)),
+ m_ConstInt(NotMask)),
+ m_Value(C))))) {
+ if (NotMask.isAllOnes() &&
+ (MaskWide == InnerLHS || MaskWide == InnerRHS)) {
SDValue WideB = MaskWide == InnerLHS ? InnerRHS : InnerLHS;
return DAG.getNode(AArch64ISD::BSP, DL, VT,
DAG.getNode(ISD::TRUNCATE, DL, VT, MaskWide),
diff --git a/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll b/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
index 4aed4c1d6b392..849a33108bbf8 100644
--- a/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
+++ b/llvm/test/CodeGen/AArch64/bsp-vselect-i1-sext.ll
@@ -258,5 +258,263 @@ entry:
%r1 = insertvalue { <4 x i16>, <4 x i32> } %r0, <4 x i32> %wand, 1
ret { <4 x i16>, <4 x i32> } %r1
}
+
+;; v4i32 integer source operands - same BSP shape built from icmp inputs.
+
+define <4 x i32> @bsp_v4i1_select_mask_sext_int(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LE-LABEL: bsp_v4i1_select_mask_sext_int:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: cmgt v3.4s, v1.4s, v0.4s
+; CHECK-LE-NEXT: cmgt v4.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: cmge v2.4s, v2.4s, #0
+; CHECK-LE-NEXT: xtn v1.4h, v3.4s
+; CHECK-LE-NEXT: xtn v3.4h, v4.4s
+; CHECK-LE-NEXT: mvn v0.16b, v0.16b
+; CHECK-LE-NEXT: xtn v2.4h, v2.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: bit v1.8b, v3.8b, v2.8b
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v4i1_select_mask_sext_int:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: and v2.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: cmgt v3.4s, v1.4s, v0.4s
+; CHECK-BE-NEXT: cmgt v4.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: cmge v2.4s, v2.4s, #0
+; CHECK-BE-NEXT: xtn v1.4h, v3.4s
+; CHECK-BE-NEXT: xtn v3.4h, v4.4s
+; CHECK-BE-NEXT: mvn v0.16b, v0.16b
+; CHECK-BE-NEXT: xtn v2.4h, v2.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: bit v1.8b, v3.8b, v2.8b
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %une = icmp ne <4 x i32> %a, %b
+ %lt = icmp slt <4 x i32> %a, %b
+ %gt = icmp sgt <4 x i32> %a, %b
+ %and = and <4 x i32> %a, %b
+ %sign = icmp sge <4 x i32> %and, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i1> %gt, <4 x i1> %lt
+ %mask = and <4 x i1> %une, %sel
+ %result = sext <4 x i1> %mask to <4 x i32>
+ ret <4 x i32> %result
+}
+
+;; Mask AND with mixed-width sources: the i1 mask comes from an i16 compare
+;; while the select arms come from i32 compares, so only the select side of
+;; the outer AND goes through the trunc-hoisted shape.
+
+define <4 x i32> @bsp_v4i1_select_mixed_width_mask(<4 x i32> %a, <4 x i16> %b) {
+; CHECK-LE-LABEL: bsp_v4i1_select_mixed_width_mask:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v2.4s, v0.4s, #0
+; CHECK-LE-NEXT: cmgt v3.4s, v0.4s, #0
+; CHECK-LE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: cmeq v1.4h, v1.4h, #0
+; CHECK-LE-NEXT: xtn v3.4h, v3.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: xtn v2.4h, v2.4s
+; CHECK-LE-NEXT: bsl v0.8b, v2.8b, v3.8b
+; CHECK-LE-NEXT: bic v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_v4i1_select_mixed_width_mask:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v1.4h, v1.4h, #0
+; CHECK-BE-NEXT: cmlt v2.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmgt v3.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: xtn v3.4h, v3.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: xtn v2.4h, v2.4s
+; CHECK-BE-NEXT: bsl v0.8b, v2.8b, v3.8b
+; CHECK-BE-NEXT: bic v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %une = icmp ne <4 x i16> %b, zeroinitializer
+ %lt = icmp slt <4 x i32> %a, zeroinitializer
+ %gt = icmp sgt <4 x i32> %a, zeroinitializer
+ %sign = icmp sge <4 x i32> %a, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i1> %lt, <4 x i1> %gt
+ %mask = and <4 x i1> %une, %sel
+ %result = sext <4 x i1> %mask to <4 x i32>
+ ret <4 x i32> %result
+}
+
+;; The four tests below feed the matched shape
+;; (or (trunc (and A B)) (and (xor (trunc A) -1) C))
+;; directly and flip one pair of commutative operands at a time, so every
+;; operand order of the OR, both ANDs and the XOR is covered.
+
+;; Mask is the first operand of the wide AND: (and A B).
+define <4 x i16> @bsp_trunc_and_mask_first(<4 x i32> %m, <4 x i32> %b, <4 x i16> %c) {
+; CHECK-LE-LABEL: bsp_trunc_and_mask_first:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_trunc_and_mask_first:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+ %andw = and <4 x i32> %m, %b
+ %t1 = trunc <4 x i32> %andw to <4 x i16>
+ %tm = trunc <4 x i32> %m to <4 x i16>
+ %notm = xor <4 x i16> %tm, splat (i16 -1)
+ %a2 = and <4 x i16> %notm, %c
+ %or = or <4 x i16> %t1, %a2
+ ret <4 x i16> %or
+}
+
+;; Mask is the second operand of the wide AND: (and B A).
+define <4 x i16> @bsp_trunc_and_mask_second(<4 x i32> %m, <4 x i32> %b, <4 x i16> %c) {
+; CHECK-LE-LABEL: bsp_trunc_and_mask_second:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_trunc_and_mask_second:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+ %andw = and <4 x i32> %b, %m
+ %t1 = trunc <4 x i32> %andw to <4 x i16>
+ %tm = trunc <4 x i32> %m to <4 x i16>
+ %notm = xor <4 x i16> %tm, splat (i16 -1)
+ %a2 = and <4 x i16> %notm, %c
+ %or = or <4 x i16> %t1, %a2
+ ret <4 x i16> %or
+}
+
+;; OR operands and the operands of the narrow AND are both swapped.
+define <4 x i16> @bsp_trunc_or_and_swapped(<4 x i32> %m, <4 x i32> %b, <4 x i16> %c) {
+; CHECK-LE-LABEL: bsp_trunc_or_and_swapped:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_trunc_or_and_swapped:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+ %andw = and <4 x i32> %m, %b
+ %t1 = trunc <4 x i32> %andw to <4 x i16>
+ %tm = trunc <4 x i32> %m to <4 x i16>
+ %notm = xor <4 x i16> %tm, splat (i16 -1)
+ %a2 = and <4 x i16> %c, %notm
+ %or = or <4 x i16> %a2, %t1
+ ret <4 x i16> %or
+}
+
+;; XOR with the all-ones constant as the first operand.
+define <4 x i16> @bsp_trunc_xor_const_first(<4 x i32> %m, <4 x i32> %b, <4 x i16> %c) {
+; CHECK-LE-LABEL: bsp_trunc_xor_const_first:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: xtn v1.4h, v1.4s
+; CHECK-LE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_trunc_xor_const_first:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: ret
+ %andw = and <4 x i32> %m, %b
+ %t1 = trunc <4 x i32> %andw to <4 x i16>
+ %tm = trunc <4 x i32> %m to <4 x i16>
+ %notm = xor <4 x i16> splat (i16 -1), %tm
+ %a2 = and <4 x i16> %notm, %c
+ %or = or <4 x i16> %t1, %a2
+ ret <4 x i16> %or
+}
+
+;; Negative test: a non-BSP pattern that must not fold. The NOT expression
+;; is not the same variable that participates in the inner AND.
+
+define <4 x i32> @bsp_nonbsp_unrelated_not(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LE-LABEL: bsp_nonbsp_unrelated_not:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: bsl v0.16b, v1.16b, v2.16b
+; CHECK-LE-NEXT: orn v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bsp_nonbsp_unrelated_not:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmge v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: bsl v0.16b, v1.16b, v2.16b
+; CHECK-BE-NEXT: orn v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %sign = icmp sge <4 x i32> %a, zeroinitializer
+ %sel = select <4 x i1> %sign, <4 x i32> %b, <4 x i32> %c
+ %not_c = xor <4 x i32> %c, splat (i32 -1)
+ %or = or <4 x i32> %sel, %not_c
+ ret <4 x i32> %or
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
More information about the llvm-commits
mailing list