[llvm] [AMDGPU] Add getCmpSelInstrCost override to re-enable SimplifyCFG speculation for vector types (PR #208043)
Frederik Harwath via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 08:06:14 PDT 2026
https://github.com/frederik-h updated https://github.com/llvm/llvm-project/pull/208043
>From 9d67adcc414ed2641efab98c72b785eeeccbaa99 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Mon, 15 Jun 2026 10:38:37 -0400
Subject: [PATCH 01/10] [SimplifyCFG] Document PHI folding threshold behavior
Starting with a generic cost model change in commit 0967957d7a94
"[CostModel] Handle all cost kinds in getCmpSelInstrCost (#148233)",
SimplifyCFG has stopped folding branches that really should be folded
on AMDGPU since not folding them causes significant spilling.
Add a test demonstrating the spilling and a simple test demonstrating
the missed fold due to the cost change.
---
.../AMDGPU/speculate-block-regpressure.ll | 197 ++++++++++++++++++
.../SimplifyCFG/AMDGPU/speculate-block.ll | 44 ++++
2 files changed, 241 insertions(+)
create mode 100644 llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
new file mode 100644
index 0000000000000..91914bec8cdb0
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s \
+; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -phi-node-folding-threshold=4 < %s \
+; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM-THRESHOLD4
+
+; Regression test for SGPR spilling caused by commit 0967957d7a94
+; "[CostModel] Handle all cost kinds in getCmpSelInstrCost".
+;
+; That commit raised the cost of vector selects for non-throughput cost kinds,
+; causing SimplifyCFG to stop speculatively executing blocks on AMDGPU. Without
+; the speculation, the branch+PHI pattern creates additional register pressure
+; at the merge point, leading to SGPR spills.
+
+; ASM: .sgpr_spill_count: 8
+; ASM-THRESHOLD4: .sgpr_spill_count: 0
+
+define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
+; IR-LABEL: define amdgpu_kernel void @reduced_fmha_kernel(
+; IR-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i32 [[ARG2:%.*]], i1 [[ARG3:%.*]], i32 [[ARG4:%.*]], i1 [[ARG5:%.*]], i1 [[ARG6:%.*]], i1 [[ARG7:%.*]], i1 [[ARG8:%.*]], i32 [[ARG9:%.*]], i1 [[ARG10:%.*]], <4 x float> [[ARG11:%.*]], i32 [[ARG12:%.*]], i32 [[ARG13:%.*]], i1 [[ARG14:%.*]], i1 [[ARG15:%.*]], i1 [[ARG16:%.*]], i1 [[ARG17:%.*]], i32 [[ARG18:%.*]], i1 [[ARG19:%.*]], i1 [[ARG20:%.*]], i32 [[ARG21:%.*]], i1 [[ARG22:%.*]], i32 [[ARG23:%.*]], i1 [[ARG24:%.*]], i1 [[ARG25:%.*]], i1 [[ARG26:%.*]], i32 [[ARG27:%.*]], i32 [[ARG28:%.*]], i1 [[ARG29:%.*]], i32 [[ARG30:%.*]], i1 [[ARG31:%.*]], i32 [[ARG32:%.*]], i1 [[ARG33:%.*]], i32 [[ARG34:%.*]], <4 x float> [[ARG35:%.*]], <4 x float> [[ARG36:%.*]], <4 x float> [[ARG37:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-NEXT: [[BB:.*]]:
+; IR-NEXT: br label %[[BB38:.*]]
+; IR: [[BB38]]:
+; IR-NEXT: [[PHI:%.*]] = phi float [ 0.000000e+00, %[[BB]] ], [ [[FMUL102:%.*]], %[[BB86:.*]] ]
+; IR-NEXT: [[PHI39:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB]] ], [ [[CALL105:%.*]], %[[BB86]] ]
+; IR-NEXT: br i1 [[ARG3]], label %[[BB40:.*]], label %[[BB86]]
+; IR: [[BB40]]:
+; IR-NEXT: [[CALL:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+; IR-NEXT: [[ICMP:%.*]] = icmp sge i32 [[ARG9]], [[CALL]]
+; IR-NEXT: [[OR:%.*]] = or i1 [[ARG10]], [[ICMP]]
+; IR-NEXT: [[CALL41:%.*]] = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+; IR-NEXT: [[ICMP42:%.*]] = icmp sge i32 [[ARG18]], [[CALL41]]
+; IR-NEXT: [[OR43:%.*]] = or i1 [[ARG19]], [[ICMP42]]
+; IR-NEXT: [[SELECT:%.*]] = select i1 [[OR43]], i1 false, i1 [[ARG3]]
+; IR-NEXT: [[SELECT44:%.*]] = select i1 [[OR]], <4 x float> zeroinitializer, <4 x float> [[ARG11]]
+; IR-NEXT: [[SELECT45:%.*]] = select i1 [[ARG14]], <4 x float> zeroinitializer, <4 x float> [[SELECT44]]
+; IR-NEXT: [[SELECT46:%.*]] = select i1 [[SELECT]], <4 x float> zeroinitializer, <4 x float> [[SELECT45]]
+; IR-NEXT: [[ICMP47:%.*]] = icmp sle i32 [[ARG21]], [[CALL]]
+; IR-NEXT: [[OR48:%.*]] = or i1 [[ICMP47]], [[ARG22]]
+; IR-NEXT: [[SELECT49:%.*]] = select i1 [[OR48]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[ICMP50:%.*]] = icmp sge i32 [[ARG23]], [[CALL41]]
+; IR-NEXT: [[ICMP51:%.*]] = icmp sle i32 [[ARG13]], [[CALL41]]
+; IR-NEXT: [[OR52:%.*]] = or i1 [[ICMP51]], [[ARG29]]
+; IR-NEXT: [[OR53:%.*]] = or i1 [[ARG8]], [[ICMP50]]
+; IR-NEXT: [[SELECT54:%.*]] = select i1 [[ARG25]], <4 x float> zeroinitializer, <4 x float> [[SELECT49]]
+; IR-NEXT: [[SELECT55:%.*]] = select i1 [[OR53]], <4 x float> zeroinitializer, <4 x float> [[SELECT54]]
+; IR-NEXT: [[SELECT56:%.*]] = select i1 [[OR52]], <4 x float> zeroinitializer, <4 x float> [[SELECT55]]
+; IR-NEXT: [[ICMP57:%.*]] = icmp sge i32 [[ARG30]], [[CALL]]
+; IR-NEXT: [[OR58:%.*]] = or i1 [[ARG6]], [[ICMP57]]
+; IR-NEXT: [[ICMP59:%.*]] = icmp sle i32 [[ARG34]], [[CALL41]]
+; IR-NEXT: [[OR60:%.*]] = or i1 [[ICMP59]], [[ARG15]]
+; IR-NEXT: [[SELECT61:%.*]] = select i1 [[OR58]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT62:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT61]]
+; IR-NEXT: [[SELECT63:%.*]] = select i1 [[OR60]], <4 x float> zeroinitializer, <4 x float> [[SELECT62]]
+; IR-NEXT: [[ICMP64:%.*]] = icmp sle i32 [[ARG]], [[CALL]]
+; IR-NEXT: [[ICMP65:%.*]] = icmp sge i32 [[ARG28]], [[CALL41]]
+; IR-NEXT: [[OR66:%.*]] = or i1 [[ARG33]], [[ICMP65]]
+; IR-NEXT: [[OR67:%.*]] = or i1 [[ICMP64]], [[ARG17]]
+; IR-NEXT: [[SELECT68:%.*]] = select i1 [[OR67]], <4 x float> zeroinitializer, <4 x float> [[ARG35]]
+; IR-NEXT: [[SELECT69:%.*]] = select i1 [[OR66]], <4 x float> zeroinitializer, <4 x float> [[SELECT68]]
+; IR-NEXT: [[SELECT70:%.*]] = select i1 [[ARG7]], <4 x float> zeroinitializer, <4 x float> [[SELECT69]]
+; IR-NEXT: [[ICMP71:%.*]] = icmp sle i32 [[ARG32]], [[CALL]]
+; IR-NEXT: [[ICMP72:%.*]] = icmp sge i32 [[ARG2]], [[CALL41]]
+; IR-NEXT: [[OR73:%.*]] = or i1 [[ARG20]], [[ICMP72]]
+; IR-NEXT: [[OR74:%.*]] = or i1 [[ICMP71]], [[ARG26]]
+; IR-NEXT: [[SELECT75:%.*]] = select i1 [[OR74]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT76:%.*]] = select i1 [[OR73]], <4 x float> zeroinitializer, <4 x float> [[SELECT75]]
+; IR-NEXT: [[SELECT77:%.*]] = select i1 [[ARG16]], <4 x float> zeroinitializer, <4 x float> [[SELECT76]]
+; IR-NEXT: [[ICMP78:%.*]] = icmp sge i32 [[ARG4]], [[CALL41]]
+; IR-NEXT: [[OR79:%.*]] = or i1 [[ARG24]], [[ICMP78]]
+; IR-NEXT: [[SELECT80:%.*]] = select i1 [[OR79]], <4 x float> zeroinitializer, <4 x float> [[ARG36]]
+; IR-NEXT: [[SELECT81:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT80]]
+; IR-NEXT: [[SELECT82:%.*]] = select i1 [[ARG6]], <4 x float> zeroinitializer, <4 x float> [[SELECT81]]
+; IR-NEXT: [[OR83:%.*]] = or i32 [[ARG1]], [[CALL]]
+; IR-NEXT: [[ICMP84:%.*]] = icmp sge i32 [[OR83]], 0
+; IR-NEXT: br i1 [[ICMP84]], label %[[BB85:.*]], label %[[BB86]]
+; IR: [[BB85]]:
+; IR-NEXT: br label %[[BB86]]
+; IR: [[BB86]]:
+; IR-NEXT: [[PHI87:%.*]] = phi <4 x float> [ [[SELECT46]], %[[BB85]] ], [ [[SELECT46]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI88:%.*]] = phi <4 x float> [ [[SELECT56]], %[[BB85]] ], [ [[SELECT56]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI89:%.*]] = phi <4 x float> [ [[SELECT63]], %[[BB85]] ], [ [[SELECT63]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI90:%.*]] = phi <4 x float> [ [[SELECT70]], %[[BB85]] ], [ [[SELECT70]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI91:%.*]] = phi <4 x float> [ [[SELECT77]], %[[BB85]] ], [ [[SELECT77]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI92:%.*]] = phi <4 x float> [ [[SELECT82]], %[[BB85]] ], [ [[ARG37]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[EXTRACTELEMENT:%.*]] = extractelement <4 x float> [[PHI87]], i64 0
+; IR-NEXT: [[EXTRACTELEMENT93:%.*]] = extractelement <4 x float> [[PHI88]], i64 0
+; IR-NEXT: [[FMUL:%.*]] = fmul float [[EXTRACTELEMENT]], [[EXTRACTELEMENT93]]
+; IR-NEXT: [[EXTRACTELEMENT94:%.*]] = extractelement <4 x float> [[PHI89]], i64 0
+; IR-NEXT: [[FMUL95:%.*]] = fmul float [[FMUL]], [[EXTRACTELEMENT94]]
+; IR-NEXT: [[EXTRACTELEMENT96:%.*]] = extractelement <4 x float> [[PHI90]], i64 0
+; IR-NEXT: [[FMUL97:%.*]] = fmul float [[FMUL95]], [[EXTRACTELEMENT96]]
+; IR-NEXT: [[EXTRACTELEMENT98:%.*]] = extractelement <4 x float> [[PHI91]], i64 0
+; IR-NEXT: [[FMUL99:%.*]] = fmul float [[FMUL97]], [[EXTRACTELEMENT98]]
+; IR-NEXT: [[EXTRACTELEMENT100:%.*]] = extractelement <4 x float> [[PHI92]], i64 0
+; IR-NEXT: [[FMUL101:%.*]] = fmul float [[FMUL99]], [[EXTRACTELEMENT100]]
+; IR-NEXT: [[FMUL102]] = fmul float [[PHI]], [[FMUL101]]
+; IR-NEXT: [[EXTRACTELEMENT103:%.*]] = extractelement <4 x float> [[PHI39]], i64 0
+; IR-NEXT: [[FMUL104:%.*]] = fmul float [[EXTRACTELEMENT103]], 0.000000e+00
+; IR-NEXT: [[FPTRUNC:%.*]] = fptrunc float [[PHI]] to half
+; IR-NEXT: [[INSERTELEMENT:%.*]] = insertelement <4 x half> zeroinitializer, half [[FPTRUNC]], i64 0
+; IR-NEXT: [[CALL105]] = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> [[INSERTELEMENT]], <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB38]]
+;
+bb:
+ br label %bb38
+
+bb38:
+ %phi = phi float [ 0.000000e+00, %bb ], [ %fmul102, %bb86 ]
+ %phi39 = phi <4 x float> [ zeroinitializer, %bb ], [ %call105, %bb86 ]
+ br i1 %arg3, label %bb40, label %bb86
+
+bb40:
+ %call = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+ %icmp = icmp sge i32 %arg9, %call
+ %or = or i1 %arg10, %icmp
+ %call41 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+ %icmp42 = icmp sge i32 %arg18, %call41
+ %or43 = or i1 %arg19, %icmp42
+ %select = select i1 %or43, i1 false, i1 %arg3
+ %select44 = select i1 %or, <4 x float> zeroinitializer, <4 x float> %arg11
+ %select45 = select i1 %arg14, <4 x float> zeroinitializer, <4 x float> %select44
+ %select46 = select i1 %select, <4 x float> zeroinitializer, <4 x float> %select45
+ %icmp47 = icmp sle i32 %arg21, %call
+ %or48 = or i1 %icmp47, %arg22
+ %select49 = select i1 %or48, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %icmp50 = icmp sge i32 %arg23, %call41
+ %icmp51 = icmp sle i32 %arg13, %call41
+ %or52 = or i1 %icmp51, %arg29
+ %or53 = or i1 %arg8, %icmp50
+ %select54 = select i1 %arg25, <4 x float> zeroinitializer, <4 x float> %select49
+ %select55 = select i1 %or53, <4 x float> zeroinitializer, <4 x float> %select54
+ %select56 = select i1 %or52, <4 x float> zeroinitializer, <4 x float> %select55
+ %icmp57 = icmp sge i32 %arg30, %call
+ %or58 = or i1 %arg6, %icmp57
+ %icmp59 = icmp sle i32 %arg34, %call41
+ %or60 = or i1 %icmp59, %arg15
+ %select61 = select i1 %or58, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select62 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select61
+ %select63 = select i1 %or60, <4 x float> zeroinitializer, <4 x float> %select62
+ %icmp64 = icmp sle i32 %arg, %call
+ %icmp65 = icmp sge i32 %arg28, %call41
+ %or66 = or i1 %arg33, %icmp65
+ %or67 = or i1 %icmp64, %arg17
+ %select68 = select i1 %or67, <4 x float> zeroinitializer, <4 x float> %arg35
+ %select69 = select i1 %or66, <4 x float> zeroinitializer, <4 x float> %select68
+ %select70 = select i1 %arg7, <4 x float> zeroinitializer, <4 x float> %select69
+ %icmp71 = icmp sle i32 %arg32, %call
+ %icmp72 = icmp sge i32 %arg2, %call41
+ %or73 = or i1 %arg20, %icmp72
+ %or74 = or i1 %icmp71, %arg26
+ %select75 = select i1 %or74, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select76 = select i1 %or73, <4 x float> zeroinitializer, <4 x float> %select75
+ %select77 = select i1 %arg16, <4 x float> zeroinitializer, <4 x float> %select76
+ %icmp78 = icmp sge i32 %arg4, %call41
+ %or79 = or i1 %arg24, %icmp78
+ %select80 = select i1 %or79, <4 x float> zeroinitializer, <4 x float> %arg36
+ %select81 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select80
+ %select82 = select i1 %arg6, <4 x float> zeroinitializer, <4 x float> %select81
+ %or83 = or i32 %arg1, %call
+ %icmp84 = icmp sge i32 %or83, 0
+ br i1 %icmp84, label %bb85, label %bb86
+
+bb85:
+ br label %bb86
+
+bb86:
+ %phi87 = phi <4 x float> [ %select46, %bb85 ], [ %select46, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi88 = phi <4 x float> [ %select56, %bb85 ], [ %select56, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi89 = phi <4 x float> [ %select63, %bb85 ], [ %select63, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi90 = phi <4 x float> [ %select70, %bb85 ], [ %select70, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi91 = phi <4 x float> [ %select77, %bb85 ], [ %select77, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi92 = phi <4 x float> [ %select82, %bb85 ], [ %arg37, %bb40 ], [ zeroinitializer, %bb38 ]
+ %extractelement = extractelement <4 x float> %phi87, i64 0
+ %extractelement93 = extractelement <4 x float> %phi88, i64 0
+ %fmul = fmul float %extractelement, %extractelement93
+ %extractelement94 = extractelement <4 x float> %phi89, i64 0
+ %fmul95 = fmul float %fmul, %extractelement94
+ %extractelement96 = extractelement <4 x float> %phi90, i64 0
+ %fmul97 = fmul float %fmul95, %extractelement96
+ %extractelement98 = extractelement <4 x float> %phi91, i64 0
+ %fmul99 = fmul float %fmul97, %extractelement98
+ %extractelement100 = extractelement <4 x float> %phi92, i64 0
+ %fmul101 = fmul float %fmul99, %extractelement100
+ %fmul102 = fmul float %phi, %fmul101
+ %extractelement103 = extractelement <4 x float> %phi39, i64 0
+ %fmul104 = fmul float %extractelement103, 0.000000e+00
+ %fptrunc = fptrunc float %phi to half
+ %insertelement = insertelement <4 x half> zeroinitializer, half %fptrunc, i64 0
+ %call105 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> %insertelement, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ br label %bb38
+}
+
+declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half>, <4 x half>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
new file mode 100644
index 0000000000000..4d889569f64ec
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -phi-node-folding-threshold=4 < %s | FileCheck %s --check-prefix=THRESHOLD4
+
+; Test that SimplifyCFG speculatively executes blocks on AMDGPU even for vector
+; types. The base cost model returns cost 4 for a <4 x float> select, so the
+; block is not speculated with the default threshold. Raising the threshold
+; to 4 allows the speculation.
+
+define amdgpu_kernel void @ham(i1 %arg3, i1 %icmp84) {
+; DEFAULT-LABEL: define amdgpu_kernel void @ham(
+; DEFAULT-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
+; DEFAULT-NEXT: [[BB:.*]]:
+; DEFAULT-NEXT: br i1 [[ARG3]], label %[[BB40:.*]], label %[[BB86:.*]]
+; DEFAULT: [[BB40]]:
+; DEFAULT-NEXT: [[ICMP841:%.*]] = icmp sge i32 0, 0
+; DEFAULT-NEXT: br i1 [[ICMP84]], label %[[BB85:.*]], label %[[BB86]]
+; DEFAULT: [[BB85]]:
+; DEFAULT-NEXT: br label %[[BB86]]
+; DEFAULT: [[BB86]]:
+; DEFAULT-NEXT: [[PHI92:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB85]] ], [ splat (float 1.000000e+00), %[[BB40]] ], [ zeroinitializer, %[[BB]] ]
+; DEFAULT-NEXT: ret void
+;
+; THRESHOLD4-LABEL: define amdgpu_kernel void @ham(
+; THRESHOLD4-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
+; THRESHOLD4-NEXT: [[BB:.*:]]
+; THRESHOLD4-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> zeroinitializer, <4 x float> splat (float 1.000000e+00)
+; THRESHOLD4-NEXT: [[SPEC_SELECT1:%.*]] = select i1 [[ARG3]], <4 x float> [[SPEC_SELECT]], <4 x float> zeroinitializer
+; THRESHOLD4-NEXT: ret void
+;
+entry:
+ br i1 %arg3, label %then, label %merge
+
+then: ; preds = %entry
+ %icmp841 = icmp sge i32 0, 0
+ br i1 %icmp84, label %then.if, label %merge
+
+then.if: ; preds = %then
+ br label %merge
+
+merge: ; preds = %then.if, %then, %entry
+ %phi92 = phi <4 x float> [ zeroinitializer, %then.if ], [ splat (float 1.000000e+00), %then ], [ zeroinitializer, %entry ]
+ ret void
+}
>From e559b4256e9e0b7825b0434bdb684650c516e8cf Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 3 Jul 2026 12:30:03 -0400
Subject: [PATCH 02/10] [AMDGPU] Return low select cost for non-throughput cost
kinds
Commit 0967957d7a94 ("[CostModel] Handle all cost kinds in
getCmpSelInstrCost") changed the base cost model to return type
legalization costs for non-throughput cost kinds. This caused
SimplifyCFG to stop folding branches to selects on AMDGPU, since
vector selects now report higher costs that exceed the folding
threshold.
Add an getCmpSelInstrCost override for AMDGPU to restore the old
behavior. This cost kind does just affect the SimplifyCFG
block speculation which is important and there seems to be
no good possibility to make a more precise decision at
this point.
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 17 ++++++++++
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 7 ++++
.../AMDGPU/speculate-block-regpressure.ll | 17 +++++-----
.../SimplifyCFG/AMDGPU/speculate-block.ll | 33 +++++--------------
4 files changed, 41 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 7631bb2dc6828..9925df32e1892 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -979,6 +979,23 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
+InstructionCost GCNTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) const {
+ // FIXME: Commit 0967957d7a94 changed the base implementation to return a
+ // cost that scales with vector width for non-throughput cost kinds. This
+ // causes SimplifyCFG's speculativelyExecuteBB (which uses TCK_SizeAndLatency)
+ // to stop folding branches to selects. We want the fold for AMDGPU, and
+ // reverting to the pre-0967957d7a94 behavior seems appropriate since
+ // SimplifyCFG lacks the context for a thorough profitability analysis.
+ if (CostKind != TTI::TCK_RecipThroughput)
+ return 1;
+
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
+}
+
InstructionCost
GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 4a239f4d6983d..647b11e2098bb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -175,6 +175,13 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr) const override;
+
bool isInlineAsmSourceOfDivergence(const CallInst *CI,
ArrayRef<unsigned> Indices = {}) const;
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
index 91914bec8cdb0..8e83438b154ee 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -13,7 +13,7 @@
; the speculation, the branch+PHI pattern creates additional register pressure
; at the merge point, leading to SGPR spills.
-; ASM: .sgpr_spill_count: 8
+; ASM: .sgpr_spill_count: 0
; ASM-THRESHOLD4: .sgpr_spill_count: 0
define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
@@ -74,16 +74,15 @@ define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i
; IR-NEXT: [[SELECT82:%.*]] = select i1 [[ARG6]], <4 x float> zeroinitializer, <4 x float> [[SELECT81]]
; IR-NEXT: [[OR83:%.*]] = or i32 [[ARG1]], [[CALL]]
; IR-NEXT: [[ICMP84:%.*]] = icmp sge i32 [[OR83]], 0
-; IR-NEXT: br i1 [[ICMP84]], label %[[BB85:.*]], label %[[BB86]]
-; IR: [[BB85]]:
+; IR-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> [[SELECT82]], <4 x float> [[ARG37]]
; IR-NEXT: br label %[[BB86]]
; IR: [[BB86]]:
-; IR-NEXT: [[PHI87:%.*]] = phi <4 x float> [ [[SELECT46]], %[[BB85]] ], [ [[SELECT46]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
-; IR-NEXT: [[PHI88:%.*]] = phi <4 x float> [ [[SELECT56]], %[[BB85]] ], [ [[SELECT56]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
-; IR-NEXT: [[PHI89:%.*]] = phi <4 x float> [ [[SELECT63]], %[[BB85]] ], [ [[SELECT63]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
-; IR-NEXT: [[PHI90:%.*]] = phi <4 x float> [ [[SELECT70]], %[[BB85]] ], [ [[SELECT70]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
-; IR-NEXT: [[PHI91:%.*]] = phi <4 x float> [ [[SELECT77]], %[[BB85]] ], [ [[SELECT77]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
-; IR-NEXT: [[PHI92:%.*]] = phi <4 x float> [ [[SELECT82]], %[[BB85]] ], [ [[ARG37]], %[[BB40]] ], [ zeroinitializer, %[[BB38]] ]
+; IR-NEXT: [[PHI87:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT46]], %[[BB40]] ]
+; IR-NEXT: [[PHI88:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT56]], %[[BB40]] ]
+; IR-NEXT: [[PHI89:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT63]], %[[BB40]] ]
+; IR-NEXT: [[PHI90:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT70]], %[[BB40]] ]
+; IR-NEXT: [[PHI91:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT77]], %[[BB40]] ]
+; IR-NEXT: [[PHI92:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SPEC_SELECT]], %[[BB40]] ]
; IR-NEXT: [[EXTRACTELEMENT:%.*]] = extractelement <4 x float> [[PHI87]], i64 0
; IR-NEXT: [[EXTRACTELEMENT93:%.*]] = extractelement <4 x float> [[PHI88]], i64 0
; IR-NEXT: [[FMUL:%.*]] = fmul float [[EXTRACTELEMENT]], [[EXTRACTELEMENT93]]
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
index 4d889569f64ec..37327b63c613a 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
@@ -1,32 +1,17 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=DEFAULT
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -phi-node-folding-threshold=4 < %s | FileCheck %s --check-prefix=THRESHOLD4
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
; Test that SimplifyCFG speculatively executes blocks on AMDGPU even for vector
-; types. The base cost model returns cost 4 for a <4 x float> select, so the
-; block is not speculated with the default threshold. Raising the threshold
-; to 4 allows the speculation.
+; types. The AMDGPU getCmpSelInstrCost override returns a low cost for selects,
+; allowing the speculation regardless of vector width.
define amdgpu_kernel void @ham(i1 %arg3, i1 %icmp84) {
-; DEFAULT-LABEL: define amdgpu_kernel void @ham(
-; DEFAULT-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
-; DEFAULT-NEXT: [[BB:.*]]:
-; DEFAULT-NEXT: br i1 [[ARG3]], label %[[BB40:.*]], label %[[BB86:.*]]
-; DEFAULT: [[BB40]]:
-; DEFAULT-NEXT: [[ICMP841:%.*]] = icmp sge i32 0, 0
-; DEFAULT-NEXT: br i1 [[ICMP84]], label %[[BB85:.*]], label %[[BB86]]
-; DEFAULT: [[BB85]]:
-; DEFAULT-NEXT: br label %[[BB86]]
-; DEFAULT: [[BB86]]:
-; DEFAULT-NEXT: [[PHI92:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB85]] ], [ splat (float 1.000000e+00), %[[BB40]] ], [ zeroinitializer, %[[BB]] ]
-; DEFAULT-NEXT: ret void
-;
-; THRESHOLD4-LABEL: define amdgpu_kernel void @ham(
-; THRESHOLD4-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
-; THRESHOLD4-NEXT: [[BB:.*:]]
-; THRESHOLD4-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> zeroinitializer, <4 x float> splat (float 1.000000e+00)
-; THRESHOLD4-NEXT: [[SPEC_SELECT1:%.*]] = select i1 [[ARG3]], <4 x float> [[SPEC_SELECT]], <4 x float> zeroinitializer
-; THRESHOLD4-NEXT: ret void
+; CHECK-LABEL: define amdgpu_kernel void @ham(
+; CHECK-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> zeroinitializer, <4 x float> splat (float 1.000000e+00)
+; CHECK-NEXT: [[SPEC_SELECT1:%.*]] = select i1 [[ARG3]], <4 x float> [[SPEC_SELECT]], <4 x float> zeroinitializer
+; CHECK-NEXT: ret void
;
entry:
br i1 %arg3, label %then, label %merge
>From 3adec7f2c17ff52cf216f908c55f43d3c9cb05dc Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 8 Jul 2026 06:39:14 -0400
Subject: [PATCH 03/10] Update test checks
---
llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll | 4 ++--
llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
index 00ce9680cab32..781eba144205d 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
@@ -24,8 +24,8 @@ define i32 @reduce_i1(i32 %arg) {
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 129 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 257 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
index eccaffc9d2a40..9e03620e2aa6f 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
@@ -24,8 +24,8 @@ define i32 @reduce_i1(i32 %arg) {
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 129 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 257 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
>From 860aa479872b8e14d315c22737f88a294b07ca59 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 8 Jul 2026 09:20:04 -0400
Subject: [PATCH 04/10] Remove additional test run
---
.../AMDGPU/speculate-block-regpressure.ll | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
index 8e83438b154ee..818fbc7928188 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -2,19 +2,15 @@
; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s --check-prefix=IR
; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s \
; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -phi-node-folding-threshold=4 < %s \
-; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM-THRESHOLD4
; Regression test for SGPR spilling caused by commit 0967957d7a94
-; "[CostModel] Handle all cost kinds in getCmpSelInstrCost".
-;
-; That commit raised the cost of vector selects for non-throughput cost kinds,
-; causing SimplifyCFG to stop speculatively executing blocks on AMDGPU. Without
-; the speculation, the branch+PHI pattern creates additional register pressure
-; at the merge point, leading to SGPR spills.
+; "[CostModel] Handle all cost kinds in getCmpSelInstrCost". That
+; commit raised the cost of vector selects for non-throughput cost
+; kinds, causing SimplifyCFG to stop speculatively executing blocks on
+; AMDGPU which increases register pressure at the merge point,
+; leading to SGPR spills.
; ASM: .sgpr_spill_count: 0
-; ASM-THRESHOLD4: .sgpr_spill_count: 0
define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
; IR-LABEL: define amdgpu_kernel void @reduced_fmha_kernel(
>From f0dfeab2714773123a5c7066f37ad306d688aeb7 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 9 Jul 2026 04:39:03 -0400
Subject: [PATCH 05/10] fixup! [AMDGPU] Return low select cost for
non-throughput cost kinds
---
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 9925df32e1892..c6e6deb84d42c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -983,12 +983,8 @@ InstructionCost GCNTTIImpl::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
TTI::OperandValueInfo Op2Info, const Instruction *I) const {
- // FIXME: Commit 0967957d7a94 changed the base implementation to return a
- // cost that scales with vector width for non-throughput cost kinds. This
- // causes SimplifyCFG's speculativelyExecuteBB (which uses TCK_SizeAndLatency)
- // to stop folding branches to selects. We want the fold for AMDGPU, and
- // reverting to the pre-0967957d7a94 behavior seems appropriate since
- // SimplifyCFG lacks the context for a thorough profitability analysis.
+ // For size and latency cost kinds, return a low cost independent of vector
+ // width to enable SimplifyCFG's speculativelyExecuteBB optimization.
if (CostKind != TTI::TCK_RecipThroughput)
return 1;
>From ace3547ffc8005e11fb4281ee6ca108fbd721d50 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 9 Jul 2026 06:54:04 -0400
Subject: [PATCH 06/10] Stop calling base implementation from
getCmpSelInstrCost
This essentially duplicates the base implementation with some
simplifications (avoid recursive call for scalars, inline call to
TargetTransformInfoImpl::getCmpSelInstrCost).
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 40 ++++++++++++++++++-
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index c6e6deb84d42c..6e4f30524b27f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -29,6 +29,7 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/KnownBits.h"
+#include <llvm/CodeGen/ISDOpcodes.h>
#include <optional>
using namespace llvm;
@@ -988,8 +989,43 @@ InstructionCost GCNTTIImpl::getCmpSelInstrCost(
if (CostKind != TTI::TCK_RecipThroughput)
return 1;
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- Op1Info, Op2Info, I);
+ // Compute cost based on type legalization.
+ const TargetLoweringBase *TLI = getTLI();
+ if (TLI->getValueType(DL, ValTy, true) == MVT::Other)
+ return 1;
+
+ const int ISD = TLI->InstructionOpcodeToISD(Opcode);
+ assert(ISD && "Invalid opcode");
+ auto getScalarCost = [TLI, ISD](std::pair<InstructionCost, MVT> ScalarLT) {
+ return TLI->isOperationExpand(ISD, ScalarLT.second) ? 1 : ScalarLT.first;
+ };
+
+ const std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
+ if (!ValTy->isVectorTy())
+ return getScalarCost(LT);
+
+ auto *ValVTy = dyn_cast_or_null<FixedVectorType>(ValTy);
+ if (!ValVTy)
+ return InstructionCost::getInvalid();
+
+ // Return legalization cost for legal vector types.
+ // TODO Handle splitting as suggested but not implemented in the base
+ // implementation?
+ if (LT.second.isVector()) {
+ const int VecISD =
+ ISD == ISD::SELECT && CondTy->isVectorTy() ? ISD::VSELECT : ISD;
+ if (!TLI->isOperationExpand(VecISD, LT.second))
+ return LT.first;
+ }
+
+ // Otherwise, assume vector needs to be scalarized.
+ InstructionCost ScalarCost =
+ getScalarCost(getTypeLegalizationCost(ValVTy->getScalarType()));
+
+ // Return scalar cost plus cost of inserting all values.
+ return getScalarizationOverhead(ValVTy, /*Insert*/ true,
+ /*Extract*/ false, CostKind) +
+ ValVTy->getNumElements() * ScalarCost;
}
InstructionCost
>From 1bd3690441eb611541f12fd7d7620ac3a896e4b2 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 30 Jul 2026 10:05:41 -0400
Subject: [PATCH 07/10] Set 'Extract' to true in scalarization overhead
computation
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 47 +++++++++----------
.../SLPVectorizer/AMDGPU/min_max.ll | 13 +++--
2 files changed, 31 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 6e4f30524b27f..00a38b3f143ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -24,10 +24,12 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/Analysis.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/Support/Casting.h"
#include "llvm/Support/KnownBits.h"
#include <llvm/CodeGen/ISDOpcodes.h>
#include <optional>
@@ -984,6 +986,9 @@ InstructionCost GCNTTIImpl::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
TTI::OperandValueInfo Op2Info, const Instruction *I) const {
+ if (isa<ScalableVectorType>(ValTy))
+ return InstructionCost::getInvalid();
+
// For size and latency cost kinds, return a low cost independent of vector
// width to enable SimplifyCFG's speculativelyExecuteBB optimization.
if (CostKind != TTI::TCK_RecipThroughput)
@@ -996,36 +1001,30 @@ InstructionCost GCNTTIImpl::getCmpSelInstrCost(
const int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
- auto getScalarCost = [TLI, ISD](std::pair<InstructionCost, MVT> ScalarLT) {
- return TLI->isOperationExpand(ISD, ScalarLT.second) ? 1 : ScalarLT.first;
- };
+ std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
- const std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
if (!ValTy->isVectorTy())
- return getScalarCost(LT);
+ return TLI->isOperationExpand(ISD, LT.second) ? 1 : LT.first;
- auto *ValVTy = dyn_cast_or_null<FixedVectorType>(ValTy);
- if (!ValVTy)
- return InstructionCost::getInvalid();
-
- // Return legalization cost for legal vector types.
- // TODO Handle splitting as suggested but not implemented in the base
- // implementation?
- if (LT.second.isVector()) {
- const int VecISD =
- ISD == ISD::SELECT && CondTy->isVectorTy() ? ISD::VSELECT : ISD;
- if (!TLI->isOperationExpand(VecISD, LT.second))
- return LT.first;
+ if (!TLI->isOperationExpand(ISD, LT.second) && !LT.second.isVector()) {
+ // The operation is legal. Assume it costs 1. Multiply
+ // by the type-legalization overhead.
+ return LT.first * 1;
}
- // Otherwise, assume vector needs to be scalarized.
- InstructionCost ScalarCost =
- getScalarCost(getTypeLegalizationCost(ValVTy->getScalarType()));
+ // Return the cost of multiple scalar invocations plus the cost of
+ // inserting and extracting the values.
+ auto *ValVTy = cast<FixedVectorType>(ValTy);
+ unsigned Num = ValVTy->getNumElements();
+ InstructionCost ScalarCost = getCmpSelInstrCost(
+ Opcode, ValTy->getScalarType(), CondTy->getScalarType(), VecPred,
+ CostKind, Op1Info, Op2Info, I);
+
+ InstructionCost Overhead =
+ getScalarizationOverhead(ValVTy, /*Insert*/ true,
+ /*Extract*/ true, CostKind);
- // Return scalar cost plus cost of inserting all values.
- return getScalarizationOverhead(ValVTy, /*Insert*/ true,
- /*Extract*/ false, CostKind) +
- ValVTy->getNumElements() * ScalarCost;
+ return Overhead + Num * ScalarCost;
}
InstructionCost
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index 57ca3db075689..8947279e9b052 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -357,17 +357,20 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
; GFX8-NEXT: bb:
; GFX8-NEXT: [[ARG0_0:%.*]] = extractelement <4 x i16> [[ARG0:%.*]], i64 0
; GFX8-NEXT: [[ARG0_1:%.*]] = extractelement <4 x i16> [[ARG0]], i64 1
+; GFX8-NEXT: [[ARG0_2:%.*]] = extractelement <4 x i16> [[ARG0]], i64 2
+; GFX8-NEXT: [[ARG0_3:%.*]] = extractelement <4 x i16> [[ARG0]], i64 3
; GFX8-NEXT: [[ARG1_0:%.*]] = extractelement <4 x i16> [[ARG1:%.*]], i64 0
; GFX8-NEXT: [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
+; GFX8-NEXT: [[ARG1_2:%.*]] = extractelement <4 x i16> [[ARG1]], i64 2
+; GFX8-NEXT: [[ARG1_3:%.*]] = extractelement <4 x i16> [[ARG1]], i64 3
; GFX8-NEXT: [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
; GFX8-NEXT: [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP3]])
+; GFX8-NEXT: [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT: [[ADD_3:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
; GFX8-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
; GFX8-NEXT: [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
-; GFX8-NEXT: [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; GFX8-NEXT: [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; GFX8-NEXT: [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX8-NEXT: [[INS_31:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
; GFX8-NEXT: ret <4 x i16> [[INS_31]]
;
; GFX9-LABEL: @uadd_sat_v4i16(
>From 61f0f7c8ef85b9e2edffba3365117e17c9e662ca Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 31 Jul 2026 06:30:51 -0400
Subject: [PATCH 08/10] Use new triple format
---
.../SimplifyCFG/AMDGPU/speculate-block-regpressure.ll | 8 ++++----
.../test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
index 818fbc7928188..0bb27af23a27a 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s --check-prefix=IR
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s \
-; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s \
+; RUN: | llc -O3 -mtriple=amdgpu9.0a-amd-amdhsa | FileCheck %s --check-prefix=ASM
; Regression test for SGPR spilling caused by commit 0967957d7a94
; "[CostModel] Handle all cost kinds in getCmpSelInstrCost". That
@@ -14,7 +14,7 @@
define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
; IR-LABEL: define amdgpu_kernel void @reduced_fmha_kernel(
-; IR-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i32 [[ARG2:%.*]], i1 [[ARG3:%.*]], i32 [[ARG4:%.*]], i1 [[ARG5:%.*]], i1 [[ARG6:%.*]], i1 [[ARG7:%.*]], i1 [[ARG8:%.*]], i32 [[ARG9:%.*]], i1 [[ARG10:%.*]], <4 x float> [[ARG11:%.*]], i32 [[ARG12:%.*]], i32 [[ARG13:%.*]], i1 [[ARG14:%.*]], i1 [[ARG15:%.*]], i1 [[ARG16:%.*]], i1 [[ARG17:%.*]], i32 [[ARG18:%.*]], i1 [[ARG19:%.*]], i1 [[ARG20:%.*]], i32 [[ARG21:%.*]], i1 [[ARG22:%.*]], i32 [[ARG23:%.*]], i1 [[ARG24:%.*]], i1 [[ARG25:%.*]], i1 [[ARG26:%.*]], i32 [[ARG27:%.*]], i32 [[ARG28:%.*]], i1 [[ARG29:%.*]], i32 [[ARG30:%.*]], i1 [[ARG31:%.*]], i32 [[ARG32:%.*]], i1 [[ARG33:%.*]], i32 [[ARG34:%.*]], <4 x float> [[ARG35:%.*]], <4 x float> [[ARG36:%.*]], <4 x float> [[ARG37:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i32 [[ARG2:%.*]], i1 [[ARG3:%.*]], i32 [[ARG4:%.*]], i1 [[ARG5:%.*]], i1 [[ARG6:%.*]], i1 [[ARG7:%.*]], i1 [[ARG8:%.*]], i32 [[ARG9:%.*]], i1 [[ARG10:%.*]], <4 x float> [[ARG11:%.*]], i32 [[ARG12:%.*]], i32 [[ARG13:%.*]], i1 [[ARG14:%.*]], i1 [[ARG15:%.*]], i1 [[ARG16:%.*]], i1 [[ARG17:%.*]], i32 [[ARG18:%.*]], i1 [[ARG19:%.*]], i1 [[ARG20:%.*]], i32 [[ARG21:%.*]], i1 [[ARG22:%.*]], i32 [[ARG23:%.*]], i1 [[ARG24:%.*]], i1 [[ARG25:%.*]], i1 [[ARG26:%.*]], i32 [[ARG27:%.*]], i32 [[ARG28:%.*]], i1 [[ARG29:%.*]], i32 [[ARG30:%.*]], i1 [[ARG31:%.*]], i32 [[ARG32:%.*]], i1 [[ARG33:%.*]], i32 [[ARG34:%.*]], <4 x float> [[ARG35:%.*]], <4 x float> [[ARG36:%.*]], <4 x float> [[ARG37:%.*]]) {
; IR-NEXT: [[BB:.*]]:
; IR-NEXT: br label %[[BB38:.*]]
; IR: [[BB38]]:
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
index 37327b63c613a..b1d4295fe7311 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
@@ -1,5 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
; Test that SimplifyCFG speculatively executes blocks on AMDGPU even for vector
; types. The AMDGPU getCmpSelInstrCost override returns a low cost for selects,
>From 7e25003127dbc33bce9d07cfd9c51ec8bd4d6fcc Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 31 Jul 2026 06:32:17 -0400
Subject: [PATCH 09/10] Add cost model test for selects
---
llvm/test/Analysis/CostModel/AMDGPU/select.ll | 185 ++++++++++++++++++
1 file changed, 185 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/select.ll
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/select.ll b/llvm/test/Analysis/CostModel/AMDGPU/select.ll
new file mode 100644
index 0000000000000..9dafcd4b39429
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/select.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=ALL %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE %s
+
+define void @select_i32(i1 %cond, i32 %a, i32 %b) {
+; ALL-LABEL: 'select_i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i32 %a, i32 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i32 %a, i32 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i32 %a, i32 %b
+ ret void
+}
+
+define void @select_i64(i1 %cond, i64 %a, i64 %b) {
+; ALL-LABEL: 'select_i64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i64 %a, i64 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i64 %a, i64 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i64 %a, i64 %b
+ ret void
+}
+
+define void @select_f32(i1 %cond, float %a, float %b) {
+; ALL-LABEL: 'select_f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, float %a, float %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, float %a, float %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, float %a, float %b
+ ret void
+}
+
+define void @select_f64(i1 %cond, double %a, double %b) {
+; ALL-LABEL: 'select_f64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, double %a, double %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, double %a, double %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, double %a, double %b
+ ret void
+}
+
+define void @select_i16(i1 %cond, i16 %a, i16 %b) {
+; ALL-LABEL: 'select_i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i16 %a, i16 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i16 %a, i16 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i16 %a, i16 %b
+ ret void
+}
+
+define void @select_f16(i1 %cond, half %a, half %b) {
+; ALL-LABEL: 'select_f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, half %a, half %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, half %a, half %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, half %a, half %b
+ ret void
+}
+
+define void @select_v2i32(i1 %cond, <2 x i32> %a, <2 x i32> %b) {
+; ALL-LABEL: 'select_v2i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+ ret void
+}
+
+define void @select_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; ALL-LABEL: 'select_v4i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+ ret void
+}
+
+define void @select_v2i16(i1 %cond, <2 x i16> %a, <2 x i16> %b) {
+; ALL-LABEL: 'select_v2i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+ ret void
+}
+
+define void @select_v4i16(i1 %cond, <4 x i16> %a, <4 x i16> %b) {
+; ALL-LABEL: 'select_v4i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+ ret void
+}
+
+define void @select_v2f32(i1 %cond, <2 x float> %a, <2 x float> %b) {
+; ALL-LABEL: 'select_v2f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+ ret void
+}
+
+define void @select_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; ALL-LABEL: 'select_v4f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+ ret void
+}
+
+define void @select_v2f16(i1 %cond, <2 x half> %a, <2 x half> %b) {
+; ALL-LABEL: 'select_v2f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+ ret void
+}
+
+define void @select_v4f16(i1 %cond, <4 x half> %a, <4 x half> %b) {
+; ALL-LABEL: 'select_v4f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+ ret void
+}
>From 38703d783c0e0d0b224b1bbd091ef7a5e4aa03cd Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 31 Jul 2026 08:17:05 -0400
Subject: [PATCH 10/10] Update test checks using update_test_checks version 6
---
.../SimplifyCFG/AMDGPU/speculate-block-regpressure.ll | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
index 0bb27af23a27a..e414b3d220864 100644
--- a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -1,4 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+
; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefix=IR
; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s \
; RUN: | llc -O3 -mtriple=amdgpu9.0a-amd-amdhsa | FileCheck %s --check-prefix=ASM
More information about the llvm-commits
mailing list