[llvm-branch-commits] [llvm] [AArch64][TTI] Cost the Ands of a NEON bit select as free (PR #232787)
Valeriy Savchenko via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 19 04:59:48 PDT 2026
https://github.com/SavchenkoValeriy updated https://github.com/llvm/llvm-project/pull/232787
>From 3802d455b888cdf57d3535740e226622f7e5464a Mon Sep 17 00:00:00 2001
From: Valeriy Savchenko <vsavchenko at apple.com>
Date: Fri, 16 Oct 2026 11:40:47 +0100
Subject: [PATCH 1/2] [AArch64][TTI] Add pre-commit tests (NFC)
---
llvm/test/Analysis/CostModel/AArch64/bsl.ll | 184 ++++++++++++++++++++
1 file changed, 184 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/AArch64/bsl.ll
diff --git a/llvm/test/Analysis/CostModel/AArch64/bsl.ll b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
new file mode 100644
index 0000000000000..2101bd9d8f750
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
@@ -0,0 +1,184 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output %s | FileCheck %s --check-prefix=COST
+; RUN: llc -mtriple=aarch64 -mattr=+sve %s -o - | FileCheck %s --check-prefix=CODE
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; TODO: The Not is folded into BSL and should be free.
+define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) {
+; COST-LABEL: 'bsl'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: bsl:
+; CODE: // %bb.0:
+; CODE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ ret <8 x i8> %r
+}
+
+; TODO: The Not should be free here as well
+define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d, ptr %p, ptr %q) {
+; COST-LABEL: 'bsl_shared_mask'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <16 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x0 = and <16 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y0 = and <16 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r0 = or <16 x i8> %x0, %y0
+; COST-NEXT: Cost Model: Found costs of 1 for: %x1 = and <16 x i8> %m, %c
+; COST-NEXT: Cost Model: Found costs of 1 for: %y1 = and <16 x i8> %n, %d
+; COST-NEXT: Cost Model: Found costs of 1 for: %r1 = or <16 x i8> %x1, %y1
+; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r0, ptr %p, align 16
+; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r1, ptr %q, align 16
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CODE-LABEL: bsl_shared_mask:
+; CODE: // %bb.0:
+; CODE-NEXT: bif v1.16b, v2.16b, v0.16b
+; CODE-NEXT: bsl v0.16b, v3.16b, v4.16b
+; CODE-NEXT: str q1, [x0]
+; CODE-NEXT: str q0, [x1]
+; CODE-NEXT: ret
+ %n = xor <16 x i8> %m, splat (i8 -1)
+ %x0 = and <16 x i8> %m, %a
+ %y0 = and <16 x i8> %n, %b
+ %r0 = or <16 x i8> %x0, %y0
+ %x1 = and <16 x i8> %m, %c
+ %y1 = and <16 x i8> %n, %d
+ %r1 = or <16 x i8> %x1, %y1
+ store <16 x i8> %r0, ptr %p
+ store <16 x i8> %r1, ptr %q
+ ret void
+}
+
+define <8 x i8> @bsl_not_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) {
+; COST-LABEL: 'bsl_not_multiuse'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %n, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: bsl_not_multiuse:
+; CODE: // %bb.0:
+; CODE-NEXT: bif v1.8b, v2.8b, v0.8b
+; CODE-NEXT: mvn v2.8b, v0.8b
+; CODE-NEXT: str d2, [x0]
+; CODE-NEXT: fmov d0, d1
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %n, ptr %p
+ ret <8 x i8> %r
+}
+
+; TODO: BIT is still formed, so %y and the Not should be free.
+define <8 x i8> @neg_bsl_and_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) {
+; COST-LABEL: 'neg_bsl_and_multiuse'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %x, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: neg_bsl_and_multiuse:
+; CODE: // %bb.0:
+; CODE-NEXT: bit v2.8b, v1.8b, v0.8b
+; CODE-NEXT: and v1.8b, v0.8b, v1.8b
+; CODE-NEXT: str d1, [x0]
+; CODE-NEXT: fmov d0, d2
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %x, ptr %p
+ ret <8 x i8> %r
+}
+
+define <8 x i8> @neg_bsl_other_block(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, i1 %f, ptr %p) {
+; COST-LABEL: 'neg_bsl_other_block'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %f, label %t, label %e
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %r, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %a
+;
+; CODE-LABEL: neg_bsl_other_block:
+; CODE: // %bb.0: // %entry
+; CODE-NEXT: tbz w0, #0, .LBB4_2
+; CODE-NEXT: // %bb.1: // %t
+; CODE-NEXT: and v1.8b, v0.8b, v1.8b
+; CODE-NEXT: bic v0.8b, v2.8b, v0.8b
+; CODE-NEXT: orr v1.8b, v1.8b, v0.8b
+; CODE-NEXT: str d1, [x1]
+; CODE-NEXT: .LBB4_2: // %common.ret
+; CODE-NEXT: fmov d0, d1
+; CODE-NEXT: ret
+entry:
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ br i1 %f, label %t, label %e
+t:
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %r, ptr %p
+ ret <8 x i8> %r
+e:
+ ret <8 x i8> %a
+}
+
+define i32 @neg_bsl_scalar(i32 %m, i32 %a, i32 %b) {
+; COST-LABEL: 'neg_bsl_scalar'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor i32 %m, -1
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and i32 %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and i32 %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or i32 %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %r
+;
+; CODE-LABEL: neg_bsl_scalar:
+; CODE: // %bb.0:
+; CODE-NEXT: and w8, w0, w1
+; CODE-NEXT: bic w9, w2, w0
+; CODE-NEXT: orr w0, w8, w9
+; CODE-NEXT: ret
+ %n = xor i32 %m, -1
+ %x = and i32 %m, %a
+ %y = and i32 %n, %b
+ %r = or i32 %x, %y
+ ret i32 %r
+}
+
+define <vscale x 16 x i8> @neg_bsl_scalable(<vscale x 16 x i8> %m, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; COST-LABEL: 'neg_bsl_scalable'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <vscale x 16 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <vscale x 16 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <vscale x 16 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <vscale x 16 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 16 x i8> %r
+;
+; CODE-LABEL: neg_bsl_scalable:
+; CODE: // %bb.0:
+; CODE-NEXT: and z1.d, z0.d, z1.d
+; CODE-NEXT: bic z0.d, z2.d, z0.d
+; CODE-NEXT: orr z0.d, z1.d, z0.d
+; CODE-NEXT: ret
+ %n = xor <vscale x 16 x i8> %m, splat (i8 -1)
+ %x = and <vscale x 16 x i8> %m, %a
+ %y = and <vscale x 16 x i8> %n, %b
+ %r = or <vscale x 16 x i8> %x, %y
+ ret <vscale x 16 x i8> %r
+}
>From da81aa84e01f735147095b350b1c38e1959e0c1e Mon Sep 17 00:00:00 2001
From: Valeriy Savchenko <vsavchenko at apple.com>
Date: Fri, 16 Oct 2026 13:28:13 +0100
Subject: [PATCH 2/2] [AArch64][TTI] Cost the Ands of a NEON bit select as free
Or(And(M, A), And(Not(M), B)) selects to a single BSL/BIT/BIF. Charge the
Or and treat the single-use Ands in its block as free, using the matcher
shared with operand sinking.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 10 ++++++++++
llvm/test/Analysis/CostModel/AArch64/bsl.ll | 16 ++++++++--------
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b631ca5ac9442..561cca222b26f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5123,6 +5123,16 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
if (!is_contained({ISD::ADD, ISD::SUB, ISD::MUL}, ISD))
return InstructionCost::getInvalid();
+ if (CtxI && CtxI->getOpcode() == Opcode && !CtxI->use_empty() &&
+ useNeonVector(Ty)) {
+ // The Ands of a bit select are folded into the BSL that the Or becomes.
+ Instruction *MainAnd, *OtherAnd;
+ Value *A, *B;
+ if (Opcode == Instruction::And &&
+ matchBSL(CtxI->user_back(), MainAnd, OtherAnd, A, B))
+ return 0;
+ }
+
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
diff --git a/llvm/test/Analysis/CostModel/AArch64/bsl.ll b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
index 2101bd9d8f750..80a03e7307ac2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/bsl.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
@@ -8,8 +8,8 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) {
; COST-LABEL: 'bsl'
; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
-; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
-; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b
; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
;
@@ -28,11 +28,11 @@ define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) {
define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d, ptr %p, ptr %q) {
; COST-LABEL: 'bsl_shared_mask'
; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <16 x i8> %m, splat (i8 -1)
-; COST-NEXT: Cost Model: Found costs of 1 for: %x0 = and <16 x i8> %m, %a
-; COST-NEXT: Cost Model: Found costs of 1 for: %y0 = and <16 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 0 for: %x0 = and <16 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y0 = and <16 x i8> %n, %b
; COST-NEXT: Cost Model: Found costs of 1 for: %r0 = or <16 x i8> %x0, %y0
-; COST-NEXT: Cost Model: Found costs of 1 for: %x1 = and <16 x i8> %m, %c
-; COST-NEXT: Cost Model: Found costs of 1 for: %y1 = and <16 x i8> %n, %d
+; COST-NEXT: Cost Model: Found costs of 0 for: %x1 = and <16 x i8> %m, %c
+; COST-NEXT: Cost Model: Found costs of 0 for: %y1 = and <16 x i8> %n, %d
; COST-NEXT: Cost Model: Found costs of 1 for: %r1 = or <16 x i8> %x1, %y1
; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r0, ptr %p, align 16
; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r1, ptr %q, align 16
@@ -60,8 +60,8 @@ define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8>
define <8 x i8> @bsl_not_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) {
; COST-LABEL: 'bsl_not_multiuse'
; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
-; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
-; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b
; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %n, ptr %p, align 8
; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
More information about the llvm-branch-commits
mailing list