[llvm] [AMDGPU] Add getCmpSelInstrCost override to re-enable SimplifyCFG speculation for vector types (PR #208043)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 08:46:33 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Frederik Harwath (frederik-h)
<details>
<summary>Changes</summary>
Commit ["[CostModel] Handle all cost kinds in getCmpSelInstrCost"](https://github.com/llvm/llvm-project/commit/0967957d7a94e1b5c749c6e963bdca25f3c6d749) changed the base cost model to consider the type in the costs for non-throughput cost kinds. This caused SimplifyCFG to stop folding branches to selects on AMDGPU, since
vector selects now report higher costs that exceed the folding threshold.
Add an getCmpSelInstrCost override for AMDGPU to restore the old behavior. This cost kind does just affect the SimplifyCFG
block speculation which is important and there seems to be no good possibility to make a more precise decision at this point.
---
Patch is 20.66 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208043.diff
6 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp (+17)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h (+7)
- (modified) llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll (+2-2)
- (modified) llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll (+2-2)
- (added) llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll (+192)
- (added) llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll (+29)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 7631bb2dc6828..9925df32e1892 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -979,6 +979,23 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
+InstructionCost GCNTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) const {
+ // FIXME: Commit 0967957d7a94 changed the base implementation to return a
+ // cost that scales with vector width for non-throughput cost kinds. This
+ // causes SimplifyCFG's speculativelyExecuteBB (which uses TCK_SizeAndLatency)
+ // to stop folding branches to selects. We want the fold for AMDGPU, and
+ // reverting to the pre-0967957d7a94 behavior seems appropriate since
+ // SimplifyCFG lacks the context for a thorough profitability analysis.
+ if (CostKind != TTI::TCK_RecipThroughput)
+ return 1;
+
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
+}
+
InstructionCost
GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 4a239f4d6983d..647b11e2098bb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -175,6 +175,13 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr) const override;
+
bool isInlineAsmSourceOfDivergence(const CallInst *CI,
ArrayRef<unsigned> Indices = {}) const;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
index 00ce9680cab32..781eba144205d 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-and.ll
@@ -24,8 +24,8 @@ define i32 @reduce_i1(i32 %arg) {
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 129 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 257 for instruction: %V256 = call i1 @llvm.vector.reduce.and.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
index eccaffc9d2a40..9e03620e2aa6f 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/reduce-or.ll
@@ -24,8 +24,8 @@ define i32 @reduce_i1(i32 %arg) {
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 65 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 130 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 260 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 129 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 257 for instruction: %V256 = call i1 @llvm.vector.reduce.or.v256i1(<256 x i1> undef)
; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
;
%V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
new file mode 100644
index 0000000000000..818fbc7928188
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s \
+; RUN: | llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a | FileCheck %s --check-prefix=ASM
+
+; Regression test for SGPR spilling caused by commit 0967957d7a94
+; "[CostModel] Handle all cost kinds in getCmpSelInstrCost". That
+; commit raised the cost of vector selects for non-throughput cost
+; kinds, causing SimplifyCFG to stop speculatively executing blocks on
+; AMDGPU which increases register pressure at the merge point,
+; leading to SGPR spills.
+
+; ASM: .sgpr_spill_count: 0
+
+define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
+; IR-LABEL: define amdgpu_kernel void @reduced_fmha_kernel(
+; IR-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i32 [[ARG2:%.*]], i1 [[ARG3:%.*]], i32 [[ARG4:%.*]], i1 [[ARG5:%.*]], i1 [[ARG6:%.*]], i1 [[ARG7:%.*]], i1 [[ARG8:%.*]], i32 [[ARG9:%.*]], i1 [[ARG10:%.*]], <4 x float> [[ARG11:%.*]], i32 [[ARG12:%.*]], i32 [[ARG13:%.*]], i1 [[ARG14:%.*]], i1 [[ARG15:%.*]], i1 [[ARG16:%.*]], i1 [[ARG17:%.*]], i32 [[ARG18:%.*]], i1 [[ARG19:%.*]], i1 [[ARG20:%.*]], i32 [[ARG21:%.*]], i1 [[ARG22:%.*]], i32 [[ARG23:%.*]], i1 [[ARG24:%.*]], i1 [[ARG25:%.*]], i1 [[ARG26:%.*]], i32 [[ARG27:%.*]], i32 [[ARG28:%.*]], i1 [[ARG29:%.*]], i32 [[ARG30:%.*]], i1 [[ARG31:%.*]], i32 [[ARG32:%.*]], i1 [[ARG33:%.*]], i32 [[ARG34:%.*]], <4 x float> [[ARG35:%.*]], <4 x float> [[ARG36:%.*]], <4 x float> [[ARG37:%.*]]) #[[ATTR0:[0-9]+]] {
+; IR-NEXT: [[BB:.*]]:
+; IR-NEXT: br label %[[BB38:.*]]
+; IR: [[BB38]]:
+; IR-NEXT: [[PHI:%.*]] = phi float [ 0.000000e+00, %[[BB]] ], [ [[FMUL102:%.*]], %[[BB86:.*]] ]
+; IR-NEXT: [[PHI39:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB]] ], [ [[CALL105:%.*]], %[[BB86]] ]
+; IR-NEXT: br i1 [[ARG3]], label %[[BB40:.*]], label %[[BB86]]
+; IR: [[BB40]]:
+; IR-NEXT: [[CALL:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+; IR-NEXT: [[ICMP:%.*]] = icmp sge i32 [[ARG9]], [[CALL]]
+; IR-NEXT: [[OR:%.*]] = or i1 [[ARG10]], [[ICMP]]
+; IR-NEXT: [[CALL41:%.*]] = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+; IR-NEXT: [[ICMP42:%.*]] = icmp sge i32 [[ARG18]], [[CALL41]]
+; IR-NEXT: [[OR43:%.*]] = or i1 [[ARG19]], [[ICMP42]]
+; IR-NEXT: [[SELECT:%.*]] = select i1 [[OR43]], i1 false, i1 [[ARG3]]
+; IR-NEXT: [[SELECT44:%.*]] = select i1 [[OR]], <4 x float> zeroinitializer, <4 x float> [[ARG11]]
+; IR-NEXT: [[SELECT45:%.*]] = select i1 [[ARG14]], <4 x float> zeroinitializer, <4 x float> [[SELECT44]]
+; IR-NEXT: [[SELECT46:%.*]] = select i1 [[SELECT]], <4 x float> zeroinitializer, <4 x float> [[SELECT45]]
+; IR-NEXT: [[ICMP47:%.*]] = icmp sle i32 [[ARG21]], [[CALL]]
+; IR-NEXT: [[OR48:%.*]] = or i1 [[ICMP47]], [[ARG22]]
+; IR-NEXT: [[SELECT49:%.*]] = select i1 [[OR48]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[ICMP50:%.*]] = icmp sge i32 [[ARG23]], [[CALL41]]
+; IR-NEXT: [[ICMP51:%.*]] = icmp sle i32 [[ARG13]], [[CALL41]]
+; IR-NEXT: [[OR52:%.*]] = or i1 [[ICMP51]], [[ARG29]]
+; IR-NEXT: [[OR53:%.*]] = or i1 [[ARG8]], [[ICMP50]]
+; IR-NEXT: [[SELECT54:%.*]] = select i1 [[ARG25]], <4 x float> zeroinitializer, <4 x float> [[SELECT49]]
+; IR-NEXT: [[SELECT55:%.*]] = select i1 [[OR53]], <4 x float> zeroinitializer, <4 x float> [[SELECT54]]
+; IR-NEXT: [[SELECT56:%.*]] = select i1 [[OR52]], <4 x float> zeroinitializer, <4 x float> [[SELECT55]]
+; IR-NEXT: [[ICMP57:%.*]] = icmp sge i32 [[ARG30]], [[CALL]]
+; IR-NEXT: [[OR58:%.*]] = or i1 [[ARG6]], [[ICMP57]]
+; IR-NEXT: [[ICMP59:%.*]] = icmp sle i32 [[ARG34]], [[CALL41]]
+; IR-NEXT: [[OR60:%.*]] = or i1 [[ICMP59]], [[ARG15]]
+; IR-NEXT: [[SELECT61:%.*]] = select i1 [[OR58]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT62:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT61]]
+; IR-NEXT: [[SELECT63:%.*]] = select i1 [[OR60]], <4 x float> zeroinitializer, <4 x float> [[SELECT62]]
+; IR-NEXT: [[ICMP64:%.*]] = icmp sle i32 [[ARG]], [[CALL]]
+; IR-NEXT: [[ICMP65:%.*]] = icmp sge i32 [[ARG28]], [[CALL41]]
+; IR-NEXT: [[OR66:%.*]] = or i1 [[ARG33]], [[ICMP65]]
+; IR-NEXT: [[OR67:%.*]] = or i1 [[ICMP64]], [[ARG17]]
+; IR-NEXT: [[SELECT68:%.*]] = select i1 [[OR67]], <4 x float> zeroinitializer, <4 x float> [[ARG35]]
+; IR-NEXT: [[SELECT69:%.*]] = select i1 [[OR66]], <4 x float> zeroinitializer, <4 x float> [[SELECT68]]
+; IR-NEXT: [[SELECT70:%.*]] = select i1 [[ARG7]], <4 x float> zeroinitializer, <4 x float> [[SELECT69]]
+; IR-NEXT: [[ICMP71:%.*]] = icmp sle i32 [[ARG32]], [[CALL]]
+; IR-NEXT: [[ICMP72:%.*]] = icmp sge i32 [[ARG2]], [[CALL41]]
+; IR-NEXT: [[OR73:%.*]] = or i1 [[ARG20]], [[ICMP72]]
+; IR-NEXT: [[OR74:%.*]] = or i1 [[ICMP71]], [[ARG26]]
+; IR-NEXT: [[SELECT75:%.*]] = select i1 [[OR74]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT76:%.*]] = select i1 [[OR73]], <4 x float> zeroinitializer, <4 x float> [[SELECT75]]
+; IR-NEXT: [[SELECT77:%.*]] = select i1 [[ARG16]], <4 x float> zeroinitializer, <4 x float> [[SELECT76]]
+; IR-NEXT: [[ICMP78:%.*]] = icmp sge i32 [[ARG4]], [[CALL41]]
+; IR-NEXT: [[OR79:%.*]] = or i1 [[ARG24]], [[ICMP78]]
+; IR-NEXT: [[SELECT80:%.*]] = select i1 [[OR79]], <4 x float> zeroinitializer, <4 x float> [[ARG36]]
+; IR-NEXT: [[SELECT81:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT80]]
+; IR-NEXT: [[SELECT82:%.*]] = select i1 [[ARG6]], <4 x float> zeroinitializer, <4 x float> [[SELECT81]]
+; IR-NEXT: [[OR83:%.*]] = or i32 [[ARG1]], [[CALL]]
+; IR-NEXT: [[ICMP84:%.*]] = icmp sge i32 [[OR83]], 0
+; IR-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> [[SELECT82]], <4 x float> [[ARG37]]
+; IR-NEXT: br label %[[BB86]]
+; IR: [[BB86]]:
+; IR-NEXT: [[PHI87:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT46]], %[[BB40]] ]
+; IR-NEXT: [[PHI88:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT56]], %[[BB40]] ]
+; IR-NEXT: [[PHI89:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT63]], %[[BB40]] ]
+; IR-NEXT: [[PHI90:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT70]], %[[BB40]] ]
+; IR-NEXT: [[PHI91:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT77]], %[[BB40]] ]
+; IR-NEXT: [[PHI92:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SPEC_SELECT]], %[[BB40]] ]
+; IR-NEXT: [[EXTRACTELEMENT:%.*]] = extractelement <4 x float> [[PHI87]], i64 0
+; IR-NEXT: [[EXTRACTELEMENT93:%.*]] = extractelement <4 x float> [[PHI88]], i64 0
+; IR-NEXT: [[FMUL:%.*]] = fmul float [[EXTRACTELEMENT]], [[EXTRACTELEMENT93]]
+; IR-NEXT: [[EXTRACTELEMENT94:%.*]] = extractelement <4 x float> [[PHI89]], i64 0
+; IR-NEXT: [[FMUL95:%.*]] = fmul float [[FMUL]], [[EXTRACTELEMENT94]]
+; IR-NEXT: [[EXTRACTELEMENT96:%.*]] = extractelement <4 x float> [[PHI90]], i64 0
+; IR-NEXT: [[FMUL97:%.*]] = fmul float [[FMUL95]], [[EXTRACTELEMENT96]]
+; IR-NEXT: [[EXTRACTELEMENT98:%.*]] = extractelement <4 x float> [[PHI91]], i64 0
+; IR-NEXT: [[FMUL99:%.*]] = fmul float [[FMUL97]], [[EXTRACTELEMENT98]]
+; IR-NEXT: [[EXTRACTELEMENT100:%.*]] = extractelement <4 x float> [[PHI92]], i64 0
+; IR-NEXT: [[FMUL101:%.*]] = fmul float [[FMUL99]], [[EXTRACTELEMENT100]]
+; IR-NEXT: [[FMUL102]] = fmul float [[PHI]], [[FMUL101]]
+; IR-NEXT: [[EXTRACTELEMENT103:%.*]] = extractelement <4 x float> [[PHI39]], i64 0
+; IR-NEXT: [[FMUL104:%.*]] = fmul float [[EXTRACTELEMENT103]], 0.000000e+00
+; IR-NEXT: [[FPTRUNC:%.*]] = fptrunc float [[PHI]] to half
+; IR-NEXT: [[INSERTELEMENT:%.*]] = insertelement <4 x half> zeroinitializer, half [[FPTRUNC]], i64 0
+; IR-NEXT: [[CALL105]] = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> [[INSERTELEMENT]], <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB38]]
+;
+bb:
+ br label %bb38
+
+bb38:
+ %phi = phi float [ 0.000000e+00, %bb ], [ %fmul102, %bb86 ]
+ %phi39 = phi <4 x float> [ zeroinitializer, %bb ], [ %call105, %bb86 ]
+ br i1 %arg3, label %bb40, label %bb86
+
+bb40:
+ %call = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+ %icmp = icmp sge i32 %arg9, %call
+ %or = or i1 %arg10, %icmp
+ %call41 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+ %icmp42 = icmp sge i32 %arg18, %call41
+ %or43 = or i1 %arg19, %icmp42
+ %select = select i1 %or43, i1 false, i1 %arg3
+ %select44 = select i1 %or, <4 x float> zeroinitializer, <4 x float> %arg11
+ %select45 = select i1 %arg14, <4 x float> zeroinitializer, <4 x float> %select44
+ %select46 = select i1 %select, <4 x float> zeroinitializer, <4 x float> %select45
+ %icmp47 = icmp sle i32 %arg21, %call
+ %or48 = or i1 %icmp47, %arg22
+ %select49 = select i1 %or48, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %icmp50 = icmp sge i32 %arg23, %call41
+ %icmp51 = icmp sle i32 %arg13, %call41
+ %or52 = or i1 %icmp51, %arg29
+ %or53 = or i1 %arg8, %icmp50
+ %select54 = select i1 %arg25, <4 x float> zeroinitializer, <4 x float> %select49
+ %select55 = select i1 %or53, <4 x float> zeroinitializer, <4 x float> %select54
+ %select56 = select i1 %or52, <4 x float> zeroinitializer, <4 x float> %select55
+ %icmp57 = icmp sge i32 %arg30, %call
+ %or58 = or i1 %arg6, %icmp57
+ %icmp59 = icmp sle i32 %arg34, %call41
+ %or60 = or i1 %icmp59, %arg15
+ %select61 = select i1 %or58, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select62 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select61
+ %select63 = select i1 %or60, <4 x float> zeroinitializer, <4 x float> %select62
+ %icmp64 = icmp sle i32 %arg, %call
+ %icmp65 = icmp sge i32 %arg28, %call41
+ %or66 = or i1 %arg33, %icmp65
+ %or67 = or i1 %icmp64, %arg17
+ %select68 = select i1 %or67, <4 x float> zeroinitializer, <4 x float> %arg35
+ %select69 = select i1 %or66, <4 x float> zeroinitializer, <4 x float> %select68
+ %select70 = select i1 %arg7, <4 x float> zeroinitializer, <4 x float> %select69
+ %icmp71 = icmp sle i32 %arg32, %call
+ %icmp72 = icmp sge i32 %arg2, %call41
+ %or73 = or i1 %arg20, %icmp72
+ %or74 = or i1 %icmp71, %arg26
+ %select75 = select i1 %or74, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select76 = select i1 %or73, <4 x float> zeroinitializer, <4 x float> %select75
+ %select77 = select i1 %arg16, <4 x float> zeroinitializer, <4 x float> %select76
+ %icmp78 = icmp sge i32 %arg4, %call41
+ %or79 = or i1 %arg24, %icmp78
+ %select80 = select i1 %or79, <4 x float> zeroinitializer, <4 x float> %arg36
+ %select81 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select80
+ %select82 = select i1 %arg6, <4 x float> zeroinitializer, <4 x float> %select81
+ %or83 = or i32 %arg1, %call
+ %icmp84 = icmp sge i32 %or83, 0
+ br i1 %icmp84, label %bb85, label %bb86
+
+bb85:
+ br label %bb86
+
+bb86:
+ %phi87 = phi <4 x float> [ %select46, %bb85 ], [ %select46, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi88 = phi <4 x float> [ %select56, %bb85 ], [ %select56, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi89 = phi <4 x float> [ %select63, %bb85 ], [ %select63, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi90 = phi <4 x float> [ %select70, %bb85 ], [ %select70, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi91 = phi <4 x float> [ %select77, %bb85 ], [ %select77, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi92 = phi <4 x float> [ %select82, %bb85 ], [ %arg37, %bb40 ], [ zeroinitializer, %bb38 ]
+ %extractelement = extractelement <4 x float> %phi87, i64 0
+ %extractelement93 = extractelement <4 x float> %phi88, i64 0
+ %fmul = fmul float %extractelement, %extractelement93
+ %extractelement94 = extractelement <4 x float> %phi89, i64 0
+ %fmul95 = fmul float %fmul, %extractelement94
+ %extractelement96 = extractelement <4 x float> %phi90, i64 0
+ %fmul97 = fmul float %fmul95, %extractelement96
+ %extractelement98 = extractelement <4 x float> %phi91, i64 0
+ %fmul99 = fmul float %fmul97, %extractelement98
+ %extractelement100 = extractelement <4 x float> %phi92, i64 0
+ %fmul101 = fmul float %fmul99, %extractelement100
+ %fmul102 = fmul float %phi, %fmul101
+ %extractelement103 = extractelement <4 x float> %phi39, i64 0
+ %fmul104 = fmul float %extractelement103, 0.000000e+00
+ %fptrunc = fptrunc float %phi to half
+ %insertelement = insertelement <4 x half> zeroinitializer, half %fptrunc, i64 0
+ %call105 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> %insertelement, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ br label %bb38
+}
+
+declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half>, <4 x half>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
new file mode 100644
index 0000000000000..37327b63c613a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
+
+; Test that SimplifyCFG speculatively executes blocks on AMDGPU even for vector
+; types. The AMDGPU getCmpSelInstrCost override returns a low cost for selects,
+; allowing the speculation regardless of vector width.
+
+define amdgpu_kernel void @ham(i1 %arg3, i1 %icmp84) {
+; CHECK-LABEL: define amdgpu_kernel void @ham(
+; CHECK-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> zeroinitializer, <4 x float> splat (float 1....
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208043
More information about the llvm-commits
mailing list