[llvm] [CostModel] Estimate the codesize cost of switch (PR #163569)
Hongyu Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 27 09:15:34 PDT 2025
https://github.com/XChy updated https://github.com/llvm/llvm-project/pull/163569
>From 4ab77c479c2b512ed0bbb511d94f3e806b9143a3 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Wed, 15 Oct 2025 22:30:35 +0800
Subject: [PATCH 1/6] [X86] Estimate the codesize cost of switch
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 17 +
llvm/test/Analysis/CostModel/X86/switch.ll | 411 ++++++++++++++++++
2 files changed, 428 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/X86/switch.ll
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 3d8d0a236a3c1..22a646f10507d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6155,6 +6155,23 @@ X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind,
const Instruction *I) const {
+ if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize) {
+ unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
+ getEstimatedNumberOfCaseClusters(*cast<SwitchInst>(I), JumpTableSize,
+ nullptr, nullptr);
+ // A trivial unconditional branch.
+ if (NumSuccs == 1)
+ return TTI::TCC_Basic;
+
+ // Assume that lowering the switch block is implemented by binary search if
+ // no jump table is generated.
+ if (JumpTableSize == 0)
+ return llvm::Log2_32_Ceil(NumSuccs) * 2 * TTI::TCC_Basic;
+
+ // Indirect branch + default compare + default jump
+ return 3 * TTI::TCC_Basic;
+ }
+
if (CostKind != TTI::TCK_RecipThroughput)
return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
// Branches are assumed to be predicted.
diff --git a/llvm/test/Analysis/CostModel/X86/switch.ll b/llvm/test/Analysis/CostModel/X86/switch.ll
new file mode 100644
index 0000000000000..e668e365e899d
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/switch.ll
@@ -0,0 +1,411 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,THROUGHPUT
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=latency 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,LATENCY
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=x86_64-unknown-linux-gnu | FileCheck %s -check-prefixes=CHECK,CODESIZE
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define i32 @single_succ_switch(i32 %x) {
+; THROUGHPUT-LABEL: 'single_succ_switch'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+;
+; LATENCY-LABEL: 'single_succ_switch'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+;
+; CODESIZE-LABEL: 'single_succ_switch'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+;
+entry:
+ switch i32 %x, label %default [
+ ]
+default:
+ ret i32 1
+}
+
+define i32 @dense_switch(i32 %x) {
+; THROUGHPUT-LABEL: 'dense_switch'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 1, label %bb1
+; THROUGHPUT-NEXT: i32 2, label %bb2
+; THROUGHPUT-NEXT: i32 3, label %bb3
+; THROUGHPUT-NEXT: i32 4, label %bb4
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; LATENCY-LABEL: 'dense_switch'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 1, label %bb1
+; LATENCY-NEXT: i32 2, label %bb2
+; LATENCY-NEXT: i32 3, label %bb3
+; LATENCY-NEXT: i32 4, label %bb4
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; CODESIZE-LABEL: 'dense_switch'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 1, label %bb1
+; CODESIZE-NEXT: i32 2, label %bb2
+; CODESIZE-NEXT: i32 3, label %bb3
+; CODESIZE-NEXT: i32 4, label %bb4
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 1, label %bb1
+ i32 2, label %bb2
+ i32 3, label %bb3
+ i32 4, label %bb4
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+default:
+ unreachable
+}
+
+define i32 @sparse_switch(i32 %x) {
+; THROUGHPUT-LABEL: 'sparse_switch'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 100, label %bb1
+; THROUGHPUT-NEXT: i32 200, label %bb2
+; THROUGHPUT-NEXT: i32 300, label %bb3
+; THROUGHPUT-NEXT: i32 400, label %bb4
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; LATENCY-LABEL: 'sparse_switch'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 100, label %bb1
+; LATENCY-NEXT: i32 200, label %bb2
+; LATENCY-NEXT: i32 300, label %bb3
+; LATENCY-NEXT: i32 400, label %bb4
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; CODESIZE-LABEL: 'sparse_switch'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 100, label %bb1
+; CODESIZE-NEXT: i32 200, label %bb2
+; CODESIZE-NEXT: i32 300, label %bb3
+; CODESIZE-NEXT: i32 400, label %bb4
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 100, label %bb1
+ i32 200, label %bb2
+ i32 300, label %bb3
+ i32 400, label %bb4
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+default:
+ unreachable
+}
+
+define i32 @dense_big_switch(i32 %x) {
+; THROUGHPUT-LABEL: 'dense_big_switch'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 1, label %bb1
+; THROUGHPUT-NEXT: i32 2, label %bb2
+; THROUGHPUT-NEXT: i32 3, label %bb3
+; THROUGHPUT-NEXT: i32 4, label %bb4
+; THROUGHPUT-NEXT: i32 5, label %bb5
+; THROUGHPUT-NEXT: i32 6, label %bb6
+; THROUGHPUT-NEXT: i32 7, label %bb7
+; THROUGHPUT-NEXT: i32 8, label %bb8
+; THROUGHPUT-NEXT: i32 9, label %bb9
+; THROUGHPUT-NEXT: i32 10, label %bb10
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 6
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 7
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 8
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 9
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 10
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; LATENCY-LABEL: 'dense_big_switch'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 1, label %bb1
+; LATENCY-NEXT: i32 2, label %bb2
+; LATENCY-NEXT: i32 3, label %bb3
+; LATENCY-NEXT: i32 4, label %bb4
+; LATENCY-NEXT: i32 5, label %bb5
+; LATENCY-NEXT: i32 6, label %bb6
+; LATENCY-NEXT: i32 7, label %bb7
+; LATENCY-NEXT: i32 8, label %bb8
+; LATENCY-NEXT: i32 9, label %bb9
+; LATENCY-NEXT: i32 10, label %bb10
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; CODESIZE-LABEL: 'dense_big_switch'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 1, label %bb1
+; CODESIZE-NEXT: i32 2, label %bb2
+; CODESIZE-NEXT: i32 3, label %bb3
+; CODESIZE-NEXT: i32 4, label %bb4
+; CODESIZE-NEXT: i32 5, label %bb5
+; CODESIZE-NEXT: i32 6, label %bb6
+; CODESIZE-NEXT: i32 7, label %bb7
+; CODESIZE-NEXT: i32 8, label %bb8
+; CODESIZE-NEXT: i32 9, label %bb9
+; CODESIZE-NEXT: i32 10, label %bb10
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 1, label %bb1
+ i32 2, label %bb2
+ i32 3, label %bb3
+ i32 4, label %bb4
+ i32 5, label %bb5
+ i32 6, label %bb6
+ i32 7, label %bb7
+ i32 8, label %bb8
+ i32 9, label %bb9
+ i32 10, label %bb10
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+bb5:
+ ret i32 5
+bb6:
+ ret i32 6
+bb7:
+ ret i32 7
+bb8:
+ ret i32 8
+bb9:
+ ret i32 9
+bb10:
+ ret i32 10
+default:
+ unreachable
+}
+
+define i32 @sparse_big_switch(i32 %x) {
+; THROUGHPUT-LABEL: 'sparse_big_switch'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 100, label %bb1
+; THROUGHPUT-NEXT: i32 200, label %bb2
+; THROUGHPUT-NEXT: i32 300, label %bb3
+; THROUGHPUT-NEXT: i32 400, label %bb4
+; THROUGHPUT-NEXT: i32 500, label %bb5
+; THROUGHPUT-NEXT: i32 600, label %bb6
+; THROUGHPUT-NEXT: i32 700, label %bb7
+; THROUGHPUT-NEXT: i32 800, label %bb8
+; THROUGHPUT-NEXT: i32 900, label %bb9
+; THROUGHPUT-NEXT: i32 1000, label %bb10
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 6
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 7
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 8
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 9
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 10
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; LATENCY-LABEL: 'sparse_big_switch'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 100, label %bb1
+; LATENCY-NEXT: i32 200, label %bb2
+; LATENCY-NEXT: i32 300, label %bb3
+; LATENCY-NEXT: i32 400, label %bb4
+; LATENCY-NEXT: i32 500, label %bb5
+; LATENCY-NEXT: i32 600, label %bb6
+; LATENCY-NEXT: i32 700, label %bb7
+; LATENCY-NEXT: i32 800, label %bb8
+; LATENCY-NEXT: i32 900, label %bb9
+; LATENCY-NEXT: i32 1000, label %bb10
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+; CODESIZE-LABEL: 'sparse_big_switch'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 100, label %bb1
+; CODESIZE-NEXT: i32 200, label %bb2
+; CODESIZE-NEXT: i32 300, label %bb3
+; CODESIZE-NEXT: i32 400, label %bb4
+; CODESIZE-NEXT: i32 500, label %bb5
+; CODESIZE-NEXT: i32 600, label %bb6
+; CODESIZE-NEXT: i32 700, label %bb7
+; CODESIZE-NEXT: i32 800, label %bb8
+; CODESIZE-NEXT: i32 900, label %bb9
+; CODESIZE-NEXT: i32 1000, label %bb10
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 6
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 7
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 8
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 9
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 10
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 100, label %bb1
+ i32 200, label %bb2
+ i32 300, label %bb3
+ i32 400, label %bb4
+ i32 500, label %bb5
+ i32 600, label %bb6
+ i32 700, label %bb7
+ i32 800, label %bb8
+ i32 900, label %bb9
+ i32 1000, label %bb10
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+bb5:
+ ret i32 5
+bb6:
+ ret i32 6
+bb7:
+ ret i32 7
+bb8:
+ ret i32 8
+bb9:
+ ret i32 9
+bb10:
+ ret i32 10
+default:
+ unreachable
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From b44548c5c76036ee425332ee190512311638b5b8 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Thu, 16 Oct 2025 03:16:42 +0800
Subject: [PATCH 2/6] resolve comments
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 37 +++++
.../lib/Target/X86/X86TargetTransformInfo.cpp | 18 +--
llvm/test/Analysis/CostModel/X86/switch.ll | 150 +++++++++++++++++-
3 files changed, 182 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 76b6c8ec68c72..3ff606e5c15f2 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1365,6 +1365,43 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
InstructionCost
getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override {
+ if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize && I) {
+ const SwitchInst *SI = cast<SwitchInst>(I);
+ unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
+ auto BrCost = thisT()->getCFInstrCost(Instruction::Br, CostKind);
+ if (SI->defaultDestUnreachable())
+ NumSuccs--;
+
+ // An unreachable switch
+ if (NumSuccs == 0)
+ return TTI::TCC_Free;
+
+ // A trivial unconditional branch.
+ if (NumSuccs == 1)
+ return BrCost;
+
+ thisT()->getEstimatedNumberOfCaseClusters(*SI, JumpTableSize, nullptr,
+ nullptr);
+ Type *BoolTy = IntegerType::get(SI->getContext(), 1);
+ Type *CondTy = SI->getCondition()->getType();
+ auto CmpCost = thisT()->getCmpSelInstrCost(
+ BinaryOperator::ICmp, BoolTy, CondTy, CmpInst::ICMP_UGT, CostKind);
+
+ // Assume that lowering the switch block is implemented by binary search
+ // if no jump table is generated.
+ if (JumpTableSize == 0)
+ return llvm::Log2_32_Ceil(NumSuccs) * (CmpCost + BrCost);
+
+ // Cost for jump table: load + jump + default compare + default jump
+ Type *EntryTy = PointerType::get(SI->getContext(), 0);
+ Align Alignment = thisT()->DL.getABITypeAlign(EntryTy);
+ auto LoadCost = thisT()->getMemoryOpCost(Instruction::Load, EntryTy,
+ Alignment, 0, CostKind);
+
+ return LoadCost + BrCost +
+ (SI->defaultDestUnreachable() ? 0 : (CmpCost + BrCost));
+ }
+
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 22a646f10507d..1cdb4397a4683 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6155,22 +6155,8 @@ X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind,
const Instruction *I) const {
- if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize) {
- unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
- getEstimatedNumberOfCaseClusters(*cast<SwitchInst>(I), JumpTableSize,
- nullptr, nullptr);
- // A trivial unconditional branch.
- if (NumSuccs == 1)
- return TTI::TCC_Basic;
-
- // Assume that lowering the switch block is implemented by binary search if
- // no jump table is generated.
- if (JumpTableSize == 0)
- return llvm::Log2_32_Ceil(NumSuccs) * 2 * TTI::TCC_Basic;
-
- // Indirect branch + default compare + default jump
- return 3 * TTI::TCC_Basic;
- }
+ if (Opcode == Instruction::Switch)
+ return BaseT::getCFInstrCost(Opcode, CostKind, I);
if (CostKind != TTI::TCK_RecipThroughput)
return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
diff --git a/llvm/test/Analysis/CostModel/X86/switch.ll b/llvm/test/Analysis/CostModel/X86/switch.ll
index e668e365e899d..ae0f6fe78d8d0 100644
--- a/llvm/test/Analysis/CostModel/X86/switch.ll
+++ b/llvm/test/Analysis/CostModel/X86/switch.ll
@@ -7,7 +7,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
define i32 @single_succ_switch(i32 %x) {
; THROUGHPUT-LABEL: 'single_succ_switch'
-; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
; THROUGHPUT-NEXT: ]
; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
;
@@ -30,7 +30,7 @@ default:
define i32 @dense_switch(i32 %x) {
; THROUGHPUT-LABEL: 'dense_switch'
-; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
; THROUGHPUT-NEXT: i32 0, label %bb0
; THROUGHPUT-NEXT: i32 1, label %bb1
; THROUGHPUT-NEXT: i32 2, label %bb2
@@ -60,7 +60,7 @@ define i32 @dense_switch(i32 %x) {
; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
;
; CODESIZE-LABEL: 'dense_switch'
-; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: switch i32 %x, label %default [
; CODESIZE-NEXT: i32 0, label %bb0
; CODESIZE-NEXT: i32 1, label %bb1
; CODESIZE-NEXT: i32 2, label %bb2
@@ -96,9 +96,77 @@ default:
unreachable
}
+define i32 @dense_switch_reachable_default(i32 %x) {
+; THROUGHPUT-LABEL: 'dense_switch_reachable_default'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 1, label %bb1
+; THROUGHPUT-NEXT: i32 2, label %bb2
+; THROUGHPUT-NEXT: i32 3, label %bb3
+; THROUGHPUT-NEXT: i32 4, label %bb4
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5
+;
+; LATENCY-LABEL: 'dense_switch_reachable_default'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 1, label %bb1
+; LATENCY-NEXT: i32 2, label %bb2
+; LATENCY-NEXT: i32 3, label %bb3
+; LATENCY-NEXT: i32 4, label %bb4
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+;
+; CODESIZE-LABEL: 'dense_switch_reachable_default'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 1, label %bb1
+; CODESIZE-NEXT: i32 2, label %bb2
+; CODESIZE-NEXT: i32 3, label %bb3
+; CODESIZE-NEXT: i32 4, label %bb4
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 1, label %bb1
+ i32 2, label %bb2
+ i32 3, label %bb3
+ i32 4, label %bb4
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+default:
+ ret i32 5
+}
+
define i32 @sparse_switch(i32 %x) {
; THROUGHPUT-LABEL: 'sparse_switch'
-; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
; THROUGHPUT-NEXT: i32 0, label %bb0
; THROUGHPUT-NEXT: i32 100, label %bb1
; THROUGHPUT-NEXT: i32 200, label %bb2
@@ -164,9 +232,77 @@ default:
unreachable
}
+define i32 @sparse_switch_reachable_default(i32 %x) {
+; THROUGHPUT-LABEL: 'sparse_switch_reachable_default'
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: i32 0, label %bb0
+; THROUGHPUT-NEXT: i32 100, label %bb1
+; THROUGHPUT-NEXT: i32 200, label %bb2
+; THROUGHPUT-NEXT: i32 300, label %bb3
+; THROUGHPUT-NEXT: i32 400, label %bb4
+; THROUGHPUT-NEXT: ]
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 1
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 2
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 3
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 4
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 5
+;
+; LATENCY-LABEL: 'sparse_switch_reachable_default'
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
+; LATENCY-NEXT: i32 0, label %bb0
+; LATENCY-NEXT: i32 100, label %bb1
+; LATENCY-NEXT: i32 200, label %bb2
+; LATENCY-NEXT: i32 300, label %bb3
+; LATENCY-NEXT: i32 400, label %bb4
+; LATENCY-NEXT: ]
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+;
+; CODESIZE-LABEL: 'sparse_switch_reachable_default'
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: i32 0, label %bb0
+; CODESIZE-NEXT: i32 100, label %bb1
+; CODESIZE-NEXT: i32 200, label %bb2
+; CODESIZE-NEXT: i32 300, label %bb3
+; CODESIZE-NEXT: i32 400, label %bb4
+; CODESIZE-NEXT: ]
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 1
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 2
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 3
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 4
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 5
+;
+entry:
+ switch i32 %x, label %default [
+ i32 0, label %bb0
+ i32 100, label %bb1
+ i32 200, label %bb2
+ i32 300, label %bb3
+ i32 400, label %bb4
+ ]
+bb0:
+ ret i32 0
+bb1:
+ ret i32 1
+bb2:
+ ret i32 2
+bb3:
+ ret i32 3
+bb4:
+ ret i32 4
+default:
+ ret i32 5
+}
+
define i32 @dense_big_switch(i32 %x) {
; THROUGHPUT-LABEL: 'dense_big_switch'
-; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
; THROUGHPUT-NEXT: i32 0, label %bb0
; THROUGHPUT-NEXT: i32 1, label %bb1
; THROUGHPUT-NEXT: i32 2, label %bb2
@@ -220,7 +356,7 @@ define i32 @dense_big_switch(i32 %x) {
; LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: unreachable
;
; CODESIZE-LABEL: 'dense_big_switch'
-; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: switch i32 %x, label %default [
+; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: switch i32 %x, label %default [
; CODESIZE-NEXT: i32 0, label %bb0
; CODESIZE-NEXT: i32 1, label %bb1
; CODESIZE-NEXT: i32 2, label %bb2
@@ -288,7 +424,7 @@ default:
define i32 @sparse_big_switch(i32 %x) {
; THROUGHPUT-LABEL: 'sparse_big_switch'
-; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 0 for instruction: switch i32 %x, label %default [
+; THROUGHPUT-NEXT: Cost Model: Found an estimated cost of 1 for instruction: switch i32 %x, label %default [
; THROUGHPUT-NEXT: i32 0, label %bb0
; THROUGHPUT-NEXT: i32 100, label %bb1
; THROUGHPUT-NEXT: i32 200, label %bb2
>From f2640e086232574ceb90ad4ba541902673903011 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Thu, 16 Oct 2025 17:32:00 +0800
Subject: [PATCH 3/6] update tests
---
.../LoopVectorize/X86/predicate-switch.ll | 185 ++++++++++++++++--
.../X86/pr48844-br-to-switch-vectorization.ll | 82 +++++++-
2 files changed, 253 insertions(+), 14 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index 2aceb279d47db..e29b89ee32d3e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -288,9 +288,46 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) {
; COST-LABEL: define void @switch_all_dests_distinct(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; COST-NEXT: br label %[[VECTOR_BODY:.*]]
+; COST: [[VECTOR_BODY]]:
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer
+; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
+; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP8]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP11]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -312,7 +349,7 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -469,7 +506,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e
; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COST-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COST: [[MIDDLE_BLOCK]]:
; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -502,7 +539,7 @@ define void @switch_all_dests_distinct_variant_using_branches(ptr %start, ptr %e
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -639,9 +676,49 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) {
; COST-LABEL: define void @switch_multiple_common_dests(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; COST-NEXT: br label %[[VECTOR_BODY:.*]]
+; COST: [[VECTOR_BODY]]:
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer
+; COST-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP9:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 14)
+; COST-NEXT: [[TMP10:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 15)
+; COST-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP10]]
+; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP11]], [[TMP13]]
+; COST-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[TMP14]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP13]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP11]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP15]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -662,7 +739,7 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -790,9 +867,43 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; COST-LABEL: define void @switch4_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; COST-NEXT: br label %[[VECTOR_BODY:.*]]
+; COST: [[VECTOR_BODY]]:
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP9]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -811,7 +922,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -957,7 +1068,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end,
; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; COST: [[MIDDLE_BLOCK]]:
; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -987,7 +1098,7 @@ define void @switch_under_br_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -1116,9 +1227,51 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST-LABEL: define void @br_under_switch_default_common_dest_with_case(
; COST-SAME: ptr [[START:%.*]], ptr [[END:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
; COST-NEXT: [[ENTRY:.*]]:
+; COST-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; COST-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; COST-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -8
+; COST-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; COST-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; COST-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; COST-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; COST-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST: [[VECTOR_PH]]:
+; COST-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; COST-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; COST-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; COST-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; COST-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; COST-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COST-NEXT: br label %[[VECTOR_BODY:.*]]
+; COST: [[VECTOR_BODY]]:
+; COST-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; COST-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; COST-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[NEXT_GEP]], align 1
+; COST-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 -12)
+; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
+; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; COST-NEXT: [[TMP10:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
+; COST-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
+; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP7]]
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP13]])
+; COST-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP14]])
+; COST-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP15]])
+; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COST: [[MIDDLE_BLOCK]]:
+; COST-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; COST-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COST: [[SCALAR_PH]]:
+; COST-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
; COST-NEXT: br label %[[LOOP_HEADER:.*]]
; COST: [[LOOP_HEADER]]:
-; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COST-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; COST-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV]], align 1
; COST-NEXT: switch i64 [[L]], label %[[DEFAULT:.*]] [
; COST-NEXT: i64 -12, label %[[IF_THEN_1:.*]]
@@ -1140,7 +1293,7 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST: [[LOOP_LATCH]]:
; COST-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i64, ptr [[PTR_IV]], i64 1
; COST-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
-; COST-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]
; COST: [[EXIT]]:
; COST-NEXT: ret void
;
@@ -1433,6 +1586,14 @@ exit:
; COST: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
; COST: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
; COST: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; COST: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; COST: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; COST: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; COST: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; COST: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; COST: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; COST: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; COST: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
;.
; FORCED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; FORCED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
index dcfebe32302be..4cf27b3a8ccce 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
@@ -12,8 +12,86 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX-NEXT: entry:
; AVX-NEXT: [[I11_NOT1:%.*]] = icmp eq ptr [[START:%.*]], [[END:%.*]]
; AVX-NEXT: br i1 [[I11_NOT1]], label [[EXIT:%.*]], label [[BB12:%.*]]
+; AVX: iter.check:
+; AVX-NEXT: [[END3:%.*]] = ptrtoint ptr [[END]] to i64
+; AVX-NEXT: [[START4:%.*]] = ptrtoint ptr [[START]] to i64
+; AVX-NEXT: [[TMP0:%.*]] = add i64 [[END3]], -4
+; AVX-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START4]]
+; AVX-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
+; AVX-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; AVX-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 28
+; AVX-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[BB12_PREHEADER:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; AVX: vector.main.loop.iter.check:
+; AVX-NEXT: [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP1]], 124
+; AVX-NEXT: br i1 [[MIN_ITERS_CHECK5]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX: vector.ph:
+; AVX-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 24
+; AVX-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775776
+; AVX-NEXT: br label [[VECTOR_BODY:%.*]]
+; AVX: vector.body:
+; AVX-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT: [[TMP4:%.*]] = shl i64 [[INDEX]], 2
+; AVX-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; AVX-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 32
+; AVX-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 64
+; AVX-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 96
+; AVX-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[NEXT_GEP]], align 4
+; AVX-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
+; AVX-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4
+; AVX-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i32>, ptr [[TMP7]], align 4
+; AVX-NEXT: [[TMP8:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 -12)
+; AVX-NEXT: [[TMP9:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 -12)
+; AVX-NEXT: [[TMP10:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 -12)
+; AVX-NEXT: [[TMP11:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 -12)
+; AVX-NEXT: [[TMP12:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD]], splat (i32 13)
+; AVX-NEXT: [[TMP13:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD6]], splat (i32 13)
+; AVX-NEXT: [[TMP14:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD7]], splat (i32 13)
+; AVX-NEXT: [[TMP15:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD8]], splat (i32 13)
+; AVX-NEXT: [[TMP16:%.*]] = or <8 x i1> [[TMP8]], [[TMP12]]
+; AVX-NEXT: [[TMP17:%.*]] = or <8 x i1> [[TMP9]], [[TMP13]]
+; AVX-NEXT: [[TMP18:%.*]] = or <8 x i1> [[TMP10]], [[TMP14]]
+; AVX-NEXT: [[TMP19:%.*]] = or <8 x i1> [[TMP11]], [[TMP15]]
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[NEXT_GEP]], i32 4, <8 x i1> [[TMP16]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP5]], i32 4, <8 x i1> [[TMP17]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP6]], i32 4, <8 x i1> [[TMP18]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP7]], i32 4, <8 x i1> [[TMP19]])
+; AVX-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX: middle.block:
+; AVX-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; AVX-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX: vec.epilog.iter.check:
+; AVX-NEXT: [[TMP21:%.*]] = shl i64 [[N_VEC]], 2
+; AVX-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP21]]
+; AVX-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; AVX-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[BB12_PREHEADER]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX: vec.epilog.ph:
+; AVX-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX-NEXT: [[N_VEC10:%.*]] = and i64 [[TMP3]], 9223372036854775800
+; AVX-NEXT: [[TMP22:%.*]] = shl i64 [[N_VEC10]], 2
+; AVX-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP22]]
+; AVX-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; AVX: vec.epilog.vector.body:
+; AVX-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX11]], 2
+; AVX-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]
+; AVX-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i32>, ptr [[NEXT_GEP12]], align 4
+; AVX-NEXT: [[TMP24:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 -12)
+; AVX-NEXT: [[TMP25:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 13)
+; AVX-NEXT: [[TMP26:%.*]] = or <8 x i1> [[TMP24]], [[TMP25]]
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[NEXT_GEP12]], i32 4, <8 x i1> [[TMP26]])
+; AVX-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX11]], 8
+; AVX-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC10]]
+; AVX-NEXT: br i1 [[TMP27]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX: vec.epilog.middle.block:
+; AVX-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
+; AVX-NEXT: br i1 [[CMP_N15]], label [[EXIT]], label [[BB12_PREHEADER]]
+; AVX: bb12.preheader:
+; AVX-NEXT: [[PTR2_PH:%.*]] = phi ptr [ [[START]], [[BB12]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP23]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; AVX-NEXT: br label [[BB13:%.*]]
; AVX: bb12:
-; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[START]], [[ENTRY:%.*]] ]
+; AVX-NEXT: [[PTR2:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LATCH:%.*]] ], [ [[PTR2_PH]], [[BB12_PREHEADER]] ]
; AVX-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR2]], align 4
; AVX-NEXT: switch i32 [[VAL]], label [[LATCH]] [
; AVX-NEXT: i32 -12, label [[STORE:%.*]]
@@ -25,7 +103,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX: latch:
; AVX-NEXT: [[PTR_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR2]], i64 4
; AVX-NEXT: [[I11_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[END]]
-; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB12]]
+; AVX-NEXT: br i1 [[I11_NOT]], label [[EXIT]], label [[BB13]], !llvm.loop [[LOOP5:![0-9]+]]
; AVX: exit:
; AVX-NEXT: ret void
;
>From 4b252583128f5c21b1da16360884c9e4ceca93b4 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Fri, 24 Oct 2025 00:14:41 +0800
Subject: [PATCH 4/6] move to TTIImpl
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 26 +++++++++++++
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 37 -------------------
2 files changed, 26 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 4cd607c0d0c8d..8d32a8e62d993 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -770,6 +770,32 @@ class TargetTransformInfoImplBase {
virtual InstructionCost getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const {
+ if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize && I) {
+ const SwitchInst *SI = cast<SwitchInst>(I);
+ unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
+ if (SI->defaultDestUnreachable())
+ NumSuccs--;
+
+ // An unreachable switch
+ if (NumSuccs == 0)
+ return TTI::TCC_Free;
+
+ // A trivial unconditional branch.
+ if (NumSuccs == 1)
+ return TTI::TCC_Basic;
+
+ getEstimatedNumberOfCaseClusters(*SI, JumpTableSize, nullptr, nullptr);
+
+ // Assume that lowering the switch block is implemented by binary search
+ // if no jump table is generated.
+ if (JumpTableSize == 0)
+ return llvm::Log2_32_Ceil(NumSuccs) * 2 * TTI::TCC_Basic;
+
+ // Cost for jump table: load + jump + default compare + default jump
+ return 2 * TTI::TCC_Basic +
+ (SI->defaultDestUnreachable() ? 0 : 2 * TTI::TCC_Basic);
+ }
+
// A phi would be free, unless we're costing the throughput because it
// will require a register.
if (Opcode == Instruction::PHI && CostKind != TTI::TCK_RecipThroughput)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3ff606e5c15f2..76b6c8ec68c72 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1365,43 +1365,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
InstructionCost
getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override {
- if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize && I) {
- const SwitchInst *SI = cast<SwitchInst>(I);
- unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
- auto BrCost = thisT()->getCFInstrCost(Instruction::Br, CostKind);
- if (SI->defaultDestUnreachable())
- NumSuccs--;
-
- // An unreachable switch
- if (NumSuccs == 0)
- return TTI::TCC_Free;
-
- // A trivial unconditional branch.
- if (NumSuccs == 1)
- return BrCost;
-
- thisT()->getEstimatedNumberOfCaseClusters(*SI, JumpTableSize, nullptr,
- nullptr);
- Type *BoolTy = IntegerType::get(SI->getContext(), 1);
- Type *CondTy = SI->getCondition()->getType();
- auto CmpCost = thisT()->getCmpSelInstrCost(
- BinaryOperator::ICmp, BoolTy, CondTy, CmpInst::ICMP_UGT, CostKind);
-
- // Assume that lowering the switch block is implemented by binary search
- // if no jump table is generated.
- if (JumpTableSize == 0)
- return llvm::Log2_32_Ceil(NumSuccs) * (CmpCost + BrCost);
-
- // Cost for jump table: load + jump + default compare + default jump
- Type *EntryTy = PointerType::get(SI->getContext(), 0);
- Align Alignment = thisT()->DL.getABITypeAlign(EntryTy);
- auto LoadCost = thisT()->getMemoryOpCost(Instruction::Load, EntryTy,
- Alignment, 0, CostKind);
-
- return LoadCost + BrCost +
- (SI->defaultDestUnreachable() ? 0 : (CmpCost + BrCost));
- }
-
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
>From 70306138847ed18df0e9ad6f642a561256e9c037 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Mon, 27 Oct 2025 17:54:08 +0800
Subject: [PATCH 5/6] update tests
---
.../LoopVectorize/X86/predicate-switch.ll | 26 +++++++++----------
.../X86/pr48844-br-to-switch-vectorization.ll | 10 +++----
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
index e29b89ee32d3e..b1ae363fdfaaa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll
@@ -313,10 +313,10 @@ define void @switch_all_dests_distinct(ptr %start, ptr %end) {
; COST-NEXT: [[TMP9:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
; COST-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP8]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP7]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP6]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP11]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 1), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP8]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -705,9 +705,9 @@ define void @switch_multiple_common_dests(ptr %start, ptr %end) {
; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP10]]
; COST-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP11]], [[TMP13]]
; COST-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[TMP14]], splat (i1 true)
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP13]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP11]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP15]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP11]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -890,9 +890,9 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; COST-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 13)
; COST-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
; COST-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP7]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP6]])
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP9]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP7]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP6]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP9]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -1256,11 +1256,11 @@ define void @br_under_switch_default_common_dest_with_case(ptr %start, ptr %end,
; COST-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
; COST-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; COST-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP12]], [[TMP7]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP13]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> zeroinitializer, ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP13]])
; COST-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP14]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 42), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP14]])
; COST-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
-; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr [[NEXT_GEP]], i32 1, <4 x i1> [[TMP15]])
+; COST-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> splat (i64 2), ptr align 1 [[NEXT_GEP]], <4 x i1> [[TMP15]])
; COST-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
index 4cf27b3a8ccce..d8ce1d9be901e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
@@ -51,10 +51,10 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX-NEXT: [[TMP17:%.*]] = or <8 x i1> [[TMP9]], [[TMP13]]
; AVX-NEXT: [[TMP18:%.*]] = or <8 x i1> [[TMP10]], [[TMP14]]
; AVX-NEXT: [[TMP19:%.*]] = or <8 x i1> [[TMP11]], [[TMP15]]
-; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[NEXT_GEP]], i32 4, <8 x i1> [[TMP16]])
-; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP5]], i32 4, <8 x i1> [[TMP17]])
-; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP6]], i32 4, <8 x i1> [[TMP18]])
-; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[TMP7]], i32 4, <8 x i1> [[TMP19]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP]], <8 x i1> [[TMP16]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP5]], <8 x i1> [[TMP17]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP6]], <8 x i1> [[TMP18]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[TMP7]], <8 x i1> [[TMP19]])
; AVX-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; AVX-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; AVX-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -80,7 +80,7 @@ define dso_local void @test(ptr %start, ptr %end) #0 {
; AVX-NEXT: [[TMP24:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 -12)
; AVX-NEXT: [[TMP25:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD13]], splat (i32 13)
; AVX-NEXT: [[TMP26:%.*]] = or <8 x i1> [[TMP24]], [[TMP25]]
-; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr [[NEXT_GEP12]], i32 4, <8 x i1> [[TMP26]])
+; AVX-NEXT: tail call void @llvm.masked.store.v8i32.p0(<8 x i32> splat (i32 42), ptr align 4 [[NEXT_GEP12]], <8 x i1> [[TMP26]])
; AVX-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX11]], 8
; AVX-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC10]]
; AVX-NEXT: br i1 [[TMP27]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
>From c6efa881fa52236e6e800034cf6f69f1262f8df6 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Tue, 28 Oct 2025 00:15:17 +0800
Subject: [PATCH 6/6] general queries
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 21 +++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 8d32a8e62d993..c7876241c0f0e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -773,6 +773,7 @@ class TargetTransformInfoImplBase {
if (Opcode == Instruction::Switch && CostKind == TTI::TCK_CodeSize && I) {
const SwitchInst *SI = cast<SwitchInst>(I);
unsigned JumpTableSize, NumSuccs = I->getNumSuccessors();
+ auto BrCost = getCFInstrCost(Instruction::Br, CostKind);
if (SI->defaultDestUnreachable())
NumSuccs--;
@@ -782,18 +783,30 @@ class TargetTransformInfoImplBase {
// A trivial unconditional branch.
if (NumSuccs == 1)
- return TTI::TCC_Basic;
+ return BrCost;
getEstimatedNumberOfCaseClusters(*SI, JumpTableSize, nullptr, nullptr);
+ Type *BoolTy = IntegerType::get(SI->getContext(), 1);
+ Type *CondTy = SI->getCondition()->getType();
+ auto CmpCost = getCmpSelInstrCost(
+ BinaryOperator::ICmp, BoolTy, CondTy, CmpInst::ICMP_UGT, CostKind,
+ {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_UniformConstantValue, TTI::OP_None}, nullptr);
+
// Assume that lowering the switch block is implemented by binary search
// if no jump table is generated.
if (JumpTableSize == 0)
- return llvm::Log2_32_Ceil(NumSuccs) * 2 * TTI::TCC_Basic;
+ return llvm::Log2_32_Ceil(NumSuccs) * (CmpCost + BrCost);
// Cost for jump table: load + jump + default compare + default jump
- return 2 * TTI::TCC_Basic +
- (SI->defaultDestUnreachable() ? 0 : 2 * TTI::TCC_Basic);
+ Type *EntryTy = PointerType::get(SI->getContext(), 0);
+ Align Alignment = DL.getABITypeAlign(EntryTy);
+ auto LoadCost =
+ getMemoryOpCost(Instruction::Load, EntryTy, Alignment, 0, CostKind,
+ {TTI::OK_AnyValue, TTI::OP_None}, nullptr);
+ return LoadCost + BrCost +
+ (SI->defaultDestUnreachable() ? 0 : (CmpCost + BrCost));
}
// A phi would be free, unless we're costing the throughput because it
More information about the llvm-commits
mailing list