[llvm-branch-commits] [llvm] [AArch64][TTI] Cost the Ands of a NEON bit select as free (PR #232787)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 16 07:59:25 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Valeriy Savchenko (SavchenkoValeriy)
<details>
<summary>Changes</summary>
Or(And(M, A), And(Not(M), B)) selects to a single BSL/BIT/BIF. Charge the
Or and treat the single-use Ands in its block as free, using the matcher
shared with operand sinking.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Full diff: https://github.com/llvm/llvm-project/pull/232787.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+10)
- (added) llvm/test/Analysis/CostModel/AArch64/bsl.ll (+184)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a557d612a0077f..e4a883d3a960e6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5111,6 +5111,16 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
if (!is_contained({ISD::ADD, ISD::SUB, ISD::MUL}, ISD))
return InstructionCost::getInvalid();
+ if (CtxI && CtxI->getOpcode() == Opcode && !CtxI->use_empty() &&
+ useNeonVector(Ty)) {
+ // The Ands of a bit select are folded into the BSL that the Or becomes.
+ Instruction *MainAnd, *OtherAnd;
+ Value *A, *B;
+ if (Opcode == Instruction::And &&
+ matchBSL(CtxI->user_back(), MainAnd, OtherAnd, A, B))
+ return 0;
+ }
+
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
diff --git a/llvm/test/Analysis/CostModel/AArch64/bsl.ll b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
new file mode 100644
index 00000000000000..80a03e7307ac2c
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/bsl.ll
@@ -0,0 +1,184 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output %s | FileCheck %s --check-prefix=COST
+; RUN: llc -mtriple=aarch64 -mattr=+sve %s -o - | FileCheck %s --check-prefix=CODE
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; TODO: The Not is folded into BSL and should be free.
+define <8 x i8> @bsl(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b) {
+; COST-LABEL: 'bsl'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: bsl:
+; CODE: // %bb.0:
+; CODE-NEXT: bsl v0.8b, v1.8b, v2.8b
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ ret <8 x i8> %r
+}
+
+; TODO: The Not should be free here as well
+define void @bsl_shared_mask(<16 x i8> %m, <16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d, ptr %p, ptr %q) {
+; COST-LABEL: 'bsl_shared_mask'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <16 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 0 for: %x0 = and <16 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y0 = and <16 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r0 = or <16 x i8> %x0, %y0
+; COST-NEXT: Cost Model: Found costs of 0 for: %x1 = and <16 x i8> %m, %c
+; COST-NEXT: Cost Model: Found costs of 0 for: %y1 = and <16 x i8> %n, %d
+; COST-NEXT: Cost Model: Found costs of 1 for: %r1 = or <16 x i8> %x1, %y1
+; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r0, ptr %p, align 16
+; COST-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %r1, ptr %q, align 16
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CODE-LABEL: bsl_shared_mask:
+; CODE: // %bb.0:
+; CODE-NEXT: bif v1.16b, v2.16b, v0.16b
+; CODE-NEXT: bsl v0.16b, v3.16b, v4.16b
+; CODE-NEXT: str q1, [x0]
+; CODE-NEXT: str q0, [x1]
+; CODE-NEXT: ret
+ %n = xor <16 x i8> %m, splat (i8 -1)
+ %x0 = and <16 x i8> %m, %a
+ %y0 = and <16 x i8> %n, %b
+ %r0 = or <16 x i8> %x0, %y0
+ %x1 = and <16 x i8> %m, %c
+ %y1 = and <16 x i8> %n, %d
+ %r1 = or <16 x i8> %x1, %y1
+ store <16 x i8> %r0, ptr %p
+ store <16 x i8> %r1, ptr %q
+ ret void
+}
+
+define <8 x i8> @bsl_not_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) {
+; COST-LABEL: 'bsl_not_multiuse'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 0 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 0 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %n, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: bsl_not_multiuse:
+; CODE: // %bb.0:
+; CODE-NEXT: bif v1.8b, v2.8b, v0.8b
+; CODE-NEXT: mvn v2.8b, v0.8b
+; CODE-NEXT: str d2, [x0]
+; CODE-NEXT: fmov d0, d1
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %n, ptr %p
+ ret <8 x i8> %r
+}
+
+; TODO: BIT is still formed, so %y and the Not should be free.
+define <8 x i8> @neg_bsl_and_multiuse(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, ptr %p) {
+; COST-LABEL: 'neg_bsl_and_multiuse'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %x, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+;
+; CODE-LABEL: neg_bsl_and_multiuse:
+; CODE: // %bb.0:
+; CODE-NEXT: bit v2.8b, v1.8b, v0.8b
+; CODE-NEXT: and v1.8b, v0.8b, v1.8b
+; CODE-NEXT: str d1, [x0]
+; CODE-NEXT: fmov d0, d2
+; CODE-NEXT: ret
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %x, ptr %p
+ ret <8 x i8> %r
+}
+
+define <8 x i8> @neg_bsl_other_block(<8 x i8> %m, <8 x i8> %a, <8 x i8> %b, i1 %f, ptr %p) {
+; COST-LABEL: 'neg_bsl_other_block'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <8 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <8 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <8 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %f, label %t, label %e
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <8 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %r, ptr %p, align 8
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %r
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %a
+;
+; CODE-LABEL: neg_bsl_other_block:
+; CODE: // %bb.0: // %entry
+; CODE-NEXT: tbz w0, #0, .LBB4_2
+; CODE-NEXT: // %bb.1: // %t
+; CODE-NEXT: and v1.8b, v0.8b, v1.8b
+; CODE-NEXT: bic v0.8b, v2.8b, v0.8b
+; CODE-NEXT: orr v1.8b, v1.8b, v0.8b
+; CODE-NEXT: str d1, [x1]
+; CODE-NEXT: .LBB4_2: // %common.ret
+; CODE-NEXT: fmov d0, d1
+; CODE-NEXT: ret
+entry:
+ %n = xor <8 x i8> %m, splat (i8 -1)
+ %x = and <8 x i8> %m, %a
+ %y = and <8 x i8> %n, %b
+ br i1 %f, label %t, label %e
+t:
+ %r = or <8 x i8> %x, %y
+ store <8 x i8> %r, ptr %p
+ ret <8 x i8> %r
+e:
+ ret <8 x i8> %a
+}
+
+define i32 @neg_bsl_scalar(i32 %m, i32 %a, i32 %b) {
+; COST-LABEL: 'neg_bsl_scalar'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor i32 %m, -1
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and i32 %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and i32 %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or i32 %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %r
+;
+; CODE-LABEL: neg_bsl_scalar:
+; CODE: // %bb.0:
+; CODE-NEXT: and w8, w0, w1
+; CODE-NEXT: bic w9, w2, w0
+; CODE-NEXT: orr w0, w8, w9
+; CODE-NEXT: ret
+ %n = xor i32 %m, -1
+ %x = and i32 %m, %a
+ %y = and i32 %n, %b
+ %r = or i32 %x, %y
+ ret i32 %r
+}
+
+define <vscale x 16 x i8> @neg_bsl_scalable(<vscale x 16 x i8> %m, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; COST-LABEL: 'neg_bsl_scalable'
+; COST-NEXT: Cost Model: Found costs of 1 for: %n = xor <vscale x 16 x i8> %m, splat (i8 -1)
+; COST-NEXT: Cost Model: Found costs of 1 for: %x = and <vscale x 16 x i8> %m, %a
+; COST-NEXT: Cost Model: Found costs of 1 for: %y = and <vscale x 16 x i8> %n, %b
+; COST-NEXT: Cost Model: Found costs of 1 for: %r = or <vscale x 16 x i8> %x, %y
+; COST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 16 x i8> %r
+;
+; CODE-LABEL: neg_bsl_scalable:
+; CODE: // %bb.0:
+; CODE-NEXT: and z1.d, z0.d, z1.d
+; CODE-NEXT: bic z0.d, z2.d, z0.d
+; CODE-NEXT: orr z0.d, z1.d, z0.d
+; CODE-NEXT: ret
+ %n = xor <vscale x 16 x i8> %m, splat (i8 -1)
+ %x = and <vscale x 16 x i8> %m, %a
+ %y = and <vscale x 16 x i8> %n, %b
+ %r = or <vscale x 16 x i8> %x, %y
+ ret <vscale x 16 x i8> %r
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/232787
More information about the llvm-branch-commits
mailing list