[llvm] f0d14b7 - [AMDGPU] Add getCmpSelInstrCost override to re-enable SimplifyCFG speculation for vector types (#208043)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 23:33:20 PDT 2026
Author: Frederik Harwath
Date: 2026-10-02T08:33:12+02:00
New Revision: f0d14b73b1a586309483f81878d5c0e4d9c90cf7
URL: https://github.com/llvm/llvm-project/commit/f0d14b73b1a586309483f81878d5c0e4d9c90cf7
DIFF: https://github.com/llvm/llvm-project/commit/f0d14b73b1a586309483f81878d5c0e4d9c90cf7.diff
LOG: [AMDGPU] Add getCmpSelInstrCost override to re-enable SimplifyCFG speculation for vector types (#208043)
Commit ["[CostModel] Handle all cost kinds in
getCmpSelInstrCost"](https://github.com/llvm/llvm-project/commit/0967957d7a94e1b5c749c6e963bdca25f3c6d749)
changed the base cost model to consider the type in the costs for
non-throughput cost kinds. This caused SimplifyCFG to stop folding
branches to selects on AMDGPU, since
vector selects now report higher costs that exceed the folding
threshold.
Add an getCmpSelInstrCost override for AMDGPU to partially restore the
old behavior by returning a constant unit cost for cost kind
`TCK_SizeAndLatency`. This affects speculative execution in SimplifyCFG
and SpeculativeExecution.
Added:
llvm/test/Analysis/CostModel/AMDGPU/cmp-select.ll
llvm/test/Analysis/CostModel/AMDGPU/cmp.ll
llvm/test/Analysis/CostModel/AMDGPU/select.ll
llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-cmp-select.ll
llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 25550ffb5369d..9662be4530f78 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1022,6 +1022,19 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
+InstructionCost GCNTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) const {
+ // For size and latency cost kinds, return a low cost independent of vector
+ // width to enable SimplifyCFG's speculativelyExecuteBB optimization.
+ if (CostKind == TTI::TCK_SizeAndLatency)
+ return 1;
+
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
+}
+
// Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with
// true16; unpacking is 2.6 to 2.9.
static constexpr unsigned MaskPackCostPerElt = 4;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index ed53519ee5ae8..593aa5ebc43a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -175,6 +175,13 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr) const override;
+
bool isInlineAsmSourceOfDivergence(const CallInst *CI,
ArrayRef<unsigned> Indices = {}) const;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/cmp-select.ll b/llvm/test/Analysis/CostModel/AMDGPU/cmp-select.ll
new file mode 100644
index 0000000000000..84cc84f3a426e
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/cmp-select.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX8 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX9 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX11 %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=size-latency 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE-LATENCY %s
+
+define void @icmp_select_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; ALL-LABEL: 'icmp_select_v2i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = icmp sgt <2 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_select_v2i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = icmp sgt <2 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_select_v2i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp sgt <2 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = icmp sgt <2 x i32> %a, %b
+ %sel = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %c
+ ret void
+}
+
+define void @icmp_select_v3i32(<3 x i32> %a, <3 x i32> %b, <3 x i32> %c) {
+; ALL-LABEL: 'icmp_select_v3i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp = icmp sgt <3 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_select_v3i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp = icmp sgt <3 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_select_v3i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp sgt <3 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = icmp sgt <3 x i32> %a, %b
+ %sel = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %c
+ ret void
+}
+
+define void @icmp_select_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; ALL-LABEL: 'icmp_select_v4i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = icmp sgt <4 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_select_v4i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = icmp sgt <4 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_select_v4i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp sgt <4 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = icmp sgt <4 x i32> %a, %b
+ %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %c
+ ret void
+}
+
+define void @fcmp_select_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; ALL-LABEL: 'fcmp_select_v2f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = fcmp ogt <2 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_select_v2f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = fcmp ogt <2 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_select_v2f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = fcmp ogt <2 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = fcmp ogt <2 x float> %a, %b
+ %sel = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %c
+ ret void
+}
+
+define void @fcmp_select_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
+; ALL-LABEL: 'fcmp_select_v3f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp = fcmp ogt <3 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_select_v3f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp = fcmp ogt <3 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_select_v3f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = fcmp ogt <3 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = fcmp ogt <3 x float> %a, %b
+ %sel = select <3 x i1> %cmp, <3 x float> %a, <3 x float> %c
+ ret void
+}
+
+define void @fcmp_select_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; ALL-LABEL: 'fcmp_select_v4f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = fcmp ogt <4 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_select_v4f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = fcmp ogt <4 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_select_v4f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = fcmp ogt <4 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = fcmp ogt <4 x float> %a, %b
+ %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %c
+ ret void
+}
+
+define void @icmp_select_v2ptr(<2 x ptr> %a, <2 x ptr> %b, <2 x ptr> %c) {
+; ALL-LABEL: 'icmp_select_v2ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = icmp ugt <2 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x ptr> %a, <2 x ptr> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_select_v2ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp = icmp ugt <2 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select <2 x i1> %cmp, <2 x ptr> %a, <2 x ptr> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_select_v2ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ugt <2 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> %cmp, <2 x ptr> %a, <2 x ptr> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = icmp ugt <2 x ptr> %a, %b
+ %sel = select <2 x i1> %cmp, <2 x ptr> %a, <2 x ptr> %c
+ ret void
+}
+
+define void @icmp_select_v4ptr(<4 x ptr> %a, <4 x ptr> %b, <4 x ptr> %c) {
+; ALL-LABEL: 'icmp_select_v4ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = icmp ugt <4 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x ptr> %a, <4 x ptr> %c
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_select_v4ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp = icmp ugt <4 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select <4 x i1> %cmp, <4 x ptr> %a, <4 x ptr> %c
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_select_v4ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ugt <4 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> %cmp, <4 x ptr> %a, <4 x ptr> %c
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp = icmp ugt <4 x ptr> %a, %b
+ %sel = select <4 x i1> %cmp, <4 x ptr> %a, <4 x ptr> %c
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/cmp.ll b/llvm/test/Analysis/CostModel/AMDGPU/cmp.ll
new file mode 100644
index 0000000000000..84c5955b0eb93
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/cmp.ll
@@ -0,0 +1,446 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX8 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX9 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX11 %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=size-latency 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE-LATENCY %s
+
+define void @icmp_i32(i32 %a, i32 %b) {
+; ALL-LABEL: 'icmp_i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i32 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_ne = icmp ne i32 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i32 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_sge = icmp sge i32 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i32 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_ne = icmp ne i32 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i32 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_sge = icmp sge i32 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i32 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_ne = icmp ne i32 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i32 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_sge = icmp sge i32 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq i32 %a, %b
+ %cmp_ne = icmp ne i32 %a, %b
+ %cmp_slt = icmp slt i32 %a, %b
+ %cmp_sge = icmp sge i32 %a, %b
+ ret void
+}
+
+define void @icmp_i64(i64 %a, i64 %b) {
+; ALL-LABEL: 'icmp_i64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i64 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i64 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_i64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i64 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i64 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_i64'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i64 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i64 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq i64 %a, %b
+ %cmp_slt = icmp slt i64 %a, %b
+ ret void
+}
+
+define void @icmp_i16(i16 %a, i16 %b) {
+; ALL-LABEL: 'icmp_i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i16 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i16 %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i16 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i16 %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq i16 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt i16 %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq i16 %a, %b
+ %cmp_slt = icmp slt i16 %a, %b
+ ret void
+}
+
+define void @fcmp_f32(float %a, float %b) {
+; ALL-LABEL: 'fcmp_f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq float %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_one = fcmp one float %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt float %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oge = fcmp oge float %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq float %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_one = fcmp one float %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt float %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oge = fcmp oge float %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq float %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_one = fcmp one float %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt float %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oge = fcmp oge float %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq float %a, %b
+ %cmp_one = fcmp one float %a, %b
+ %cmp_olt = fcmp olt float %a, %b
+ %cmp_oge = fcmp oge float %a, %b
+ ret void
+}
+
+define void @fcmp_f64(double %a, double %b) {
+; ALL-LABEL: 'fcmp_f64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq double %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt double %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_f64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq double %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt double %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_f64'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq double %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt double %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq double %a, %b
+ %cmp_olt = fcmp olt double %a, %b
+ ret void
+}
+
+define void @fcmp_f16(half %a, half %b) {
+; ALL-LABEL: 'fcmp_f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq half %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt half %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq half %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt half %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq half %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt half %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq half %a, %b
+ %cmp_olt = fcmp olt half %a, %b
+ ret void
+}
+
+define void @icmp_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; ALL-LABEL: 'icmp_v2i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_eq = icmp eq <2 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_slt = icmp slt <2 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v2i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_eq = icmp eq <2 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_slt = icmp slt <2 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v2i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <2 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <2 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <2 x i32> %a, %b
+ %cmp_slt = icmp slt <2 x i32> %a, %b
+ ret void
+}
+
+define void @icmp_v3i32(<3 x i32> %a, <3 x i32> %b) {
+; ALL-LABEL: 'icmp_v3i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_eq = icmp eq <3 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_slt = icmp slt <3 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v3i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_eq = icmp eq <3 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_slt = icmp slt <3 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v3i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <3 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <3 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <3 x i32> %a, %b
+ %cmp_slt = icmp slt <3 x i32> %a, %b
+ ret void
+}
+
+define void @icmp_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; ALL-LABEL: 'icmp_v4i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_eq = icmp eq <4 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_slt = icmp slt <4 x i32> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v4i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_eq = icmp eq <4 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_slt = icmp slt <4 x i32> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v4i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <4 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <4 x i32> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <4 x i32> %a, %b
+ %cmp_slt = icmp slt <4 x i32> %a, %b
+ ret void
+}
+
+define void @icmp_v2i16(<2 x i16> %a, <2 x i16> %b) {
+; ALL-LABEL: 'icmp_v2i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_eq = icmp eq <2 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_slt = icmp slt <2 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v2i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_eq = icmp eq <2 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_slt = icmp slt <2 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v2i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <2 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <2 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <2 x i16> %a, %b
+ %cmp_slt = icmp slt <2 x i16> %a, %b
+ ret void
+}
+
+define void @icmp_v3i16(<3 x i16> %a, <3 x i16> %b) {
+; ALL-LABEL: 'icmp_v3i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_eq = icmp eq <3 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_slt = icmp slt <3 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v3i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_eq = icmp eq <3 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_slt = icmp slt <3 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v3i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <3 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <3 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <3 x i16> %a, %b
+ %cmp_slt = icmp slt <3 x i16> %a, %b
+ ret void
+}
+
+define void @icmp_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; ALL-LABEL: 'icmp_v4i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_eq = icmp eq <4 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_slt = icmp slt <4 x i16> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v4i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_eq = icmp eq <4 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_slt = icmp slt <4 x i16> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v4i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <4 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_slt = icmp slt <4 x i16> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <4 x i16> %a, %b
+ %cmp_slt = icmp slt <4 x i16> %a, %b
+ ret void
+}
+
+define void @fcmp_v2f32(<2 x float> %a, <2 x float> %b) {
+; ALL-LABEL: 'fcmp_v2f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_oeq = fcmp oeq <2 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_olt = fcmp olt <2 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v2f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_oeq = fcmp oeq <2 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_olt = fcmp olt <2 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v2f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <2 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <2 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <2 x float> %a, %b
+ %cmp_olt = fcmp olt <2 x float> %a, %b
+ ret void
+}
+
+define void @fcmp_v3f32(<3 x float> %a, <3 x float> %b) {
+; ALL-LABEL: 'fcmp_v3f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_oeq = fcmp oeq <3 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_olt = fcmp olt <3 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v3f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_oeq = fcmp oeq <3 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_olt = fcmp olt <3 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v3f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <3 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <3 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <3 x float> %a, %b
+ %cmp_olt = fcmp olt <3 x float> %a, %b
+ ret void
+}
+
+define void @fcmp_v4f32(<4 x float> %a, <4 x float> %b) {
+; ALL-LABEL: 'fcmp_v4f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_oeq = fcmp oeq <4 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_olt = fcmp olt <4 x float> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v4f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_oeq = fcmp oeq <4 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_olt = fcmp olt <4 x float> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v4f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <4 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <4 x float> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <4 x float> %a, %b
+ %cmp_olt = fcmp olt <4 x float> %a, %b
+ ret void
+}
+
+define void @fcmp_v2f16(<2 x half> %a, <2 x half> %b) {
+; ALL-LABEL: 'fcmp_v2f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_oeq = fcmp oeq <2 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_olt = fcmp olt <2 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v2f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_oeq = fcmp oeq <2 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cmp_olt = fcmp olt <2 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v2f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <2 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <2 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <2 x half> %a, %b
+ %cmp_olt = fcmp olt <2 x half> %a, %b
+ ret void
+}
+
+define void @fcmp_v3f16(<3 x half> %a, <3 x half> %b) {
+; ALL-LABEL: 'fcmp_v3f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_oeq = fcmp oeq <3 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_olt = fcmp olt <3 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v3f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_oeq = fcmp oeq <3 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %cmp_olt = fcmp olt <3 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v3f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <3 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <3 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <3 x half> %a, %b
+ %cmp_olt = fcmp olt <3 x half> %a, %b
+ ret void
+}
+
+define void @fcmp_v4f16(<4 x half> %a, <4 x half> %b) {
+; ALL-LABEL: 'fcmp_v4f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_oeq = fcmp oeq <4 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_olt = fcmp olt <4 x half> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fcmp_v4f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_oeq = fcmp oeq <4 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %cmp_olt = fcmp olt <4 x half> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'fcmp_v4f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_oeq = fcmp oeq <4 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_olt = fcmp olt <4 x half> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_oeq = fcmp oeq <4 x half> %a, %b
+ %cmp_olt = fcmp olt <4 x half> %a, %b
+ ret void
+}
+
+define void @icmp_v2ptr(<2 x ptr> %a, <2 x ptr> %b) {
+; ALL-LABEL: 'icmp_v2ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_eq = icmp eq <2 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_ugt = icmp ugt <2 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v2ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_eq = icmp eq <2 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %cmp_ugt = icmp ugt <2 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v2ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <2 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_ugt = icmp ugt <2 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <2 x ptr> %a, %b
+ %cmp_ugt = icmp ugt <2 x ptr> %a, %b
+ ret void
+}
+
+define void @icmp_v4ptr(<4 x ptr> %a, <4 x ptr> %b) {
+; ALL-LABEL: 'icmp_v4ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_eq = icmp eq <4 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_ugt = icmp ugt <4 x ptr> %a, %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'icmp_v4ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_eq = icmp eq <4 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %cmp_ugt = icmp ugt <4 x ptr> %a, %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'icmp_v4ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_eq = icmp eq <4 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cmp_ugt = icmp ugt <4 x ptr> %a, %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %cmp_eq = icmp eq <4 x ptr> %a, %b
+ %cmp_ugt = icmp ugt <4 x ptr> %a, %b
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/select.ll b/llvm/test/Analysis/CostModel/AMDGPU/select.ll
new file mode 100644
index 0000000000000..356efcf795888
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/select.ll
@@ -0,0 +1,367 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX8 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX9 %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX11 %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=size-latency 2>&1 -disable-output -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=ALL-SIZE-LATENCY %s
+
+define void @select_i32(i1 %cond, i32 %a, i32 %b) {
+; ALL-LABEL: 'select_i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i32 %a, i32 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i32 %a, i32 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i32 %a, i32 %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i32 %a, i32 %b
+ ret void
+}
+
+define void @select_i64(i1 %cond, i64 %a, i64 %b) {
+; ALL-LABEL: 'select_i64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i64 %a, i64 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i64 %a, i64 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_i64'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i64 %a, i64 %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i64 %a, i64 %b
+ ret void
+}
+
+define void @select_f32(i1 %cond, float %a, float %b) {
+; ALL-LABEL: 'select_f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, float %a, float %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, float %a, float %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, float %a, float %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, float %a, float %b
+ ret void
+}
+
+define void @select_f64(i1 %cond, double %a, double %b) {
+; ALL-LABEL: 'select_f64'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, double %a, double %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f64'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, double %a, double %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_f64'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, double %a, double %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, double %a, double %b
+ ret void
+}
+
+define void @select_i16(i1 %cond, i16 %a, i16 %b) {
+; ALL-LABEL: 'select_i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i16 %a, i16 %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i16 %a, i16 %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, i16 %a, i16 %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, i16 %a, i16 %b
+ ret void
+}
+
+define void @select_f16(i1 %cond, half %a, half %b) {
+; ALL-LABEL: 'select_f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, half %a, half %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, half %a, half %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, half %a, half %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, half %a, half %b
+ ret void
+}
+
+define void @select_ptr(i1 %cond, ptr %a, ptr %b) {
+; ALL-LABEL: 'select_ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, ptr %a, ptr %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, ptr %a, ptr %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, ptr %a, ptr %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, ptr %a, ptr %b
+ ret void
+}
+
+define void @select_v2i32(i1 %cond, <2 x i32> %a, <2 x i32> %b) {
+; ALL-LABEL: 'select_v2i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v2i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x i32> %a, <2 x i32> %b
+ ret void
+}
+
+define void @select_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; ALL-LABEL: 'select_v4i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v4i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x i32> %a, <4 x i32> %b
+ ret void
+}
+
+define void @select_v2i16(i1 %cond, <2 x i16> %a, <2 x i16> %b) {
+; ALL-LABEL: 'select_v2i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v2i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x i16> %a, <2 x i16> %b
+ ret void
+}
+
+define void @select_v4i16(i1 %cond, <4 x i16> %a, <4 x i16> %b) {
+; ALL-LABEL: 'select_v4i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v4i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x i16> %a, <4 x i16> %b
+ ret void
+}
+
+define void @select_v2f32(i1 %cond, <2 x float> %a, <2 x float> %b) {
+; ALL-LABEL: 'select_v2f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v2f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x float> %a, <2 x float> %b
+ ret void
+}
+
+define void @select_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; ALL-LABEL: 'select_v4f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v4f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x float> %a, <4 x float> %b
+ ret void
+}
+
+define void @select_v2f16(i1 %cond, <2 x half> %a, <2 x half> %b) {
+; ALL-LABEL: 'select_v2f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v2f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x half> %a, <2 x half> %b
+ ret void
+}
+
+define void @select_v4f16(i1 %cond, <4 x half> %a, <4 x half> %b) {
+; ALL-LABEL: 'select_v4f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v4f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x half> %a, <4 x half> %b
+ ret void
+}
+
+define void @select_v3i32(i1 %cond, <3 x i32> %a, <3 x i32> %b) {
+; ALL-LABEL: 'select_v3i32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select i1 %cond, <3 x i32> %a, <3 x i32> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v3i32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %sel = select i1 %cond, <3 x i32> %a, <3 x i32> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v3i32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x i32> %a, <3 x i32> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <3 x i32> %a, <3 x i32> %b
+ ret void
+}
+
+define void @select_v3f32(i1 %cond, <3 x float> %a, <3 x float> %b) {
+; ALL-LABEL: 'select_v3f32'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x float> %a, <3 x float> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v3f32'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x float> %a, <3 x float> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v3f32'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x float> %a, <3 x float> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <3 x float> %a, <3 x float> %b
+ ret void
+}
+
+define void @select_v3i16(i1 %cond, <3 x i16> %a, <3 x i16> %b) {
+; ALL-LABEL: 'select_v3i16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x i16> %a, <3 x i16> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v3i16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x i16> %a, <3 x i16> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v3i16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x i16> %a, <3 x i16> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <3 x i16> %a, <3 x i16> %b
+ ret void
+}
+
+define void @select_v3f16(i1 %cond, <3 x half> %a, <3 x half> %b) {
+; ALL-LABEL: 'select_v3f16'
+; ALL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x half> %a, <3 x half> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v3f16'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x half> %a, <3 x half> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v3f16'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <3 x half> %a, <3 x half> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <3 x half> %a, <3 x half> %b
+ ret void
+}
+
+define void @select_v2ptr(i1 %cond, <2 x ptr> %a, <2 x ptr> %b) {
+; ALL-LABEL: 'select_v2ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select i1 %cond, <2 x ptr> %a, <2 x ptr> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v2ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %sel = select i1 %cond, <2 x ptr> %a, <2 x ptr> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v2ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <2 x ptr> %a, <2 x ptr> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <2 x ptr> %a, <2 x ptr> %b
+ ret void
+}
+
+define void @select_v4ptr(i1 %cond, <4 x ptr> %a, <4 x ptr> %b) {
+; ALL-LABEL: 'select_v4ptr'
+; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <4 x ptr> %a, <4 x ptr> %b
+; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'select_v4ptr'
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %sel = select i1 %cond, <4 x ptr> %a, <4 x ptr> %b
+; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; ALL-SIZE-LATENCY-LABEL: 'select_v4ptr'
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %sel = select i1 %cond, <4 x ptr> %a, <4 x ptr> %b
+; ALL-SIZE-LATENCY-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+ %sel = select i1 %cond, <4 x ptr> %a, <4 x ptr> %b
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-cmp-select.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-cmp-select.ll
new file mode 100644
index 0000000000000..2d87c1f8119ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-cmp-select.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=loop-unroll < %s | FileCheck %s
+
+; Loop unrolling uses TCK_SizeAndLatency for non-minsize functions. Even though
+; AMDGPU reports low size-latency costs for cmp/selects to enable SimplifyCFG
+; speculation, this wide loop should still remain too expensive to unroll.
+
+define void @wide_select_loop(ptr addrspace(1) %p, <16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: define void @wide_select_loop(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], <16 x float> [[A:%.*]], <16 x float> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <16 x float>, ptr addrspace(1) [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load <16 x float>, ptr addrspace(1) [[GEP]], align 64
+; CHECK-NEXT: [[C1:%.*]] = fcmp ogt <16 x float> [[V]], [[A]]
+; CHECK-NEXT: [[S1:%.*]] = select <16 x i1> [[C1]], <16 x float> [[V]], <16 x float> [[B]]
+; CHECK-NEXT: [[C2:%.*]] = fcmp ogt <16 x float> [[S1]], [[A]]
+; CHECK-NEXT: [[S2:%.*]] = select <16 x i1> [[C2]], <16 x float> [[S1]], <16 x float> [[B]]
+; CHECK-NEXT: [[C3:%.*]] = fcmp ogt <16 x float> [[S2]], [[A]]
+; CHECK-NEXT: [[S3:%.*]] = select <16 x i1> [[C3]], <16 x float> [[S2]], <16 x float> [[B]]
+; CHECK-NEXT: [[C4:%.*]] = fcmp ogt <16 x float> [[S3]], [[A]]
+; CHECK-NEXT: [[S4:%.*]] = select <16 x i1> [[C4]], <16 x float> [[S3]], <16 x float> [[B]]
+; CHECK-NEXT: [[C5:%.*]] = fcmp ogt <16 x float> [[S4]], [[A]]
+; CHECK-NEXT: [[S5:%.*]] = select <16 x i1> [[C5]], <16 x float> [[S4]], <16 x float> [[B]]
+; CHECK-NEXT: store <16 x float> [[S5]], ptr addrspace(1) [[GEP]], align 64
+; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 64
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+ %gep = getelementptr inbounds <16 x float>, ptr addrspace(1) %p, i32 %i
+ %v = load <16 x float>, ptr addrspace(1) %gep, align 64
+ %c1 = fcmp ogt <16 x float> %v, %a
+ %s1 = select <16 x i1> %c1, <16 x float> %v, <16 x float> %b
+ %c2 = fcmp ogt <16 x float> %s1, %a
+ %s2 = select <16 x i1> %c2, <16 x float> %s1, <16 x float> %b
+ %c3 = fcmp ogt <16 x float> %s2, %a
+ %s3 = select <16 x i1> %c3, <16 x float> %s2, <16 x float> %b
+ %c4 = fcmp ogt <16 x float> %s3, %a
+ %s4 = select <16 x i1> %c4, <16 x float> %s3, <16 x float> %b
+ %c5 = fcmp ogt <16 x float> %s4, %a
+ %s5 = select <16 x i1> %c5, <16 x float> %s4, <16 x float> %b
+ store <16 x float> %s5, ptr addrspace(1) %gep, align 64
+ %inc = add nuw nsw i32 %i, 1
+ %cmp = icmp slt i32 %inc, 64
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
new file mode 100644
index 0000000000000..541f205435cb4
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block-regpressure.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefix=IR
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s \
+; RUN: | llc | FileCheck %s --check-prefix=ASM
+
+; Regression test for SGPR spilling caused by commit 0967957d7a94
+; "[CostModel] Handle all cost kinds in getCmpSelInstrCost". That
+; commit raised the cost of vector selects for non-throughput cost
+; kinds, causing SimplifyCFG to stop speculatively executing blocks on
+; AMDGPU which increases register pressure at the merge point,
+; leading to SGPR spills.
+
+; ASM: .sgpr_spill_count: 0
+
+define amdgpu_kernel void @reduced_fmha_kernel(i32 %arg, i32 %arg1, i32 %arg2, i1 %arg3, i32 %arg4, i1 %arg5, i1 %arg6, i1 %arg7, i1 %arg8, i32 %arg9, i1 %arg10, <4 x float> %arg11, i32 %arg12, i32 %arg13, i1 %arg14, i1 %arg15, i1 %arg16, i1 %arg17, i32 %arg18, i1 %arg19, i1 %arg20, i32 %arg21, i1 %arg22, i32 %arg23, i1 %arg24, i1 %arg25, i1 %arg26, i32 %arg27, i32 %arg28, i1 %arg29, i32 %arg30, i1 %arg31, i32 %arg32, i1 %arg33, i32 %arg34, <4 x float> %arg35, <4 x float> %arg36, <4 x float> %arg37) {
+; IR-LABEL: define amdgpu_kernel void @reduced_fmha_kernel(
+; IR-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i32 [[ARG2:%.*]], i1 [[ARG3:%.*]], i32 [[ARG4:%.*]], i1 [[ARG5:%.*]], i1 [[ARG6:%.*]], i1 [[ARG7:%.*]], i1 [[ARG8:%.*]], i32 [[ARG9:%.*]], i1 [[ARG10:%.*]], <4 x float> [[ARG11:%.*]], i32 [[ARG12:%.*]], i32 [[ARG13:%.*]], i1 [[ARG14:%.*]], i1 [[ARG15:%.*]], i1 [[ARG16:%.*]], i1 [[ARG17:%.*]], i32 [[ARG18:%.*]], i1 [[ARG19:%.*]], i1 [[ARG20:%.*]], i32 [[ARG21:%.*]], i1 [[ARG22:%.*]], i32 [[ARG23:%.*]], i1 [[ARG24:%.*]], i1 [[ARG25:%.*]], i1 [[ARG26:%.*]], i32 [[ARG27:%.*]], i32 [[ARG28:%.*]], i1 [[ARG29:%.*]], i32 [[ARG30:%.*]], i1 [[ARG31:%.*]], i32 [[ARG32:%.*]], i1 [[ARG33:%.*]], i32 [[ARG34:%.*]], <4 x float> [[ARG35:%.*]], <4 x float> [[ARG36:%.*]], <4 x float> [[ARG37:%.*]]) {
+; IR-NEXT: [[BB:.*]]:
+; IR-NEXT: br label %[[BB38:.*]]
+; IR: [[BB38]]:
+; IR-NEXT: [[PHI:%.*]] = phi float [ 0.000000e+00, %[[BB]] ], [ [[FMUL102:%.*]], %[[BB86:.*]] ]
+; IR-NEXT: [[PHI39:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB]] ], [ [[CALL105:%.*]], %[[BB86]] ]
+; IR-NEXT: br i1 [[ARG3]], label %[[BB40:.*]], label %[[BB86]]
+; IR: [[BB40]]:
+; IR-NEXT: [[CALL:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+; IR-NEXT: [[ICMP:%.*]] = icmp sge i32 [[ARG9]], [[CALL]]
+; IR-NEXT: [[OR:%.*]] = or i1 [[ARG10]], [[ICMP]]
+; IR-NEXT: [[CALL41:%.*]] = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+; IR-NEXT: [[ICMP42:%.*]] = icmp sge i32 [[ARG18]], [[CALL41]]
+; IR-NEXT: [[OR43:%.*]] = or i1 [[ARG19]], [[ICMP42]]
+; IR-NEXT: [[SELECT:%.*]] = select i1 [[OR43]], i1 false, i1 [[ARG3]]
+; IR-NEXT: [[SELECT44:%.*]] = select i1 [[OR]], <4 x float> zeroinitializer, <4 x float> [[ARG11]]
+; IR-NEXT: [[SELECT45:%.*]] = select i1 [[ARG14]], <4 x float> zeroinitializer, <4 x float> [[SELECT44]]
+; IR-NEXT: [[SELECT46:%.*]] = select i1 [[SELECT]], <4 x float> zeroinitializer, <4 x float> [[SELECT45]]
+; IR-NEXT: [[ICMP47:%.*]] = icmp sle i32 [[ARG21]], [[CALL]]
+; IR-NEXT: [[OR48:%.*]] = or i1 [[ICMP47]], [[ARG22]]
+; IR-NEXT: [[SELECT49:%.*]] = select i1 [[OR48]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[ICMP50:%.*]] = icmp sge i32 [[ARG23]], [[CALL41]]
+; IR-NEXT: [[ICMP51:%.*]] = icmp sle i32 [[ARG13]], [[CALL41]]
+; IR-NEXT: [[OR52:%.*]] = or i1 [[ICMP51]], [[ARG29]]
+; IR-NEXT: [[OR53:%.*]] = or i1 [[ARG8]], [[ICMP50]]
+; IR-NEXT: [[SELECT54:%.*]] = select i1 [[ARG25]], <4 x float> zeroinitializer, <4 x float> [[SELECT49]]
+; IR-NEXT: [[SELECT55:%.*]] = select i1 [[OR53]], <4 x float> zeroinitializer, <4 x float> [[SELECT54]]
+; IR-NEXT: [[SELECT56:%.*]] = select i1 [[OR52]], <4 x float> zeroinitializer, <4 x float> [[SELECT55]]
+; IR-NEXT: [[ICMP57:%.*]] = icmp sge i32 [[ARG30]], [[CALL]]
+; IR-NEXT: [[OR58:%.*]] = or i1 [[ARG6]], [[ICMP57]]
+; IR-NEXT: [[ICMP59:%.*]] = icmp sle i32 [[ARG34]], [[CALL41]]
+; IR-NEXT: [[OR60:%.*]] = or i1 [[ICMP59]], [[ARG15]]
+; IR-NEXT: [[SELECT61:%.*]] = select i1 [[OR58]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT62:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT61]]
+; IR-NEXT: [[SELECT63:%.*]] = select i1 [[OR60]], <4 x float> zeroinitializer, <4 x float> [[SELECT62]]
+; IR-NEXT: [[ICMP64:%.*]] = icmp sle i32 [[ARG]], [[CALL]]
+; IR-NEXT: [[ICMP65:%.*]] = icmp sge i32 [[ARG28]], [[CALL41]]
+; IR-NEXT: [[OR66:%.*]] = or i1 [[ARG33]], [[ICMP65]]
+; IR-NEXT: [[OR67:%.*]] = or i1 [[ICMP64]], [[ARG17]]
+; IR-NEXT: [[SELECT68:%.*]] = select i1 [[OR67]], <4 x float> zeroinitializer, <4 x float> [[ARG35]]
+; IR-NEXT: [[SELECT69:%.*]] = select i1 [[OR66]], <4 x float> zeroinitializer, <4 x float> [[SELECT68]]
+; IR-NEXT: [[SELECT70:%.*]] = select i1 [[ARG7]], <4 x float> zeroinitializer, <4 x float> [[SELECT69]]
+; IR-NEXT: [[ICMP71:%.*]] = icmp sle i32 [[ARG32]], [[CALL]]
+; IR-NEXT: [[ICMP72:%.*]] = icmp sge i32 [[ARG2]], [[CALL41]]
+; IR-NEXT: [[OR73:%.*]] = or i1 [[ARG20]], [[ICMP72]]
+; IR-NEXT: [[OR74:%.*]] = or i1 [[ICMP71]], [[ARG26]]
+; IR-NEXT: [[SELECT75:%.*]] = select i1 [[OR74]], <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+; IR-NEXT: [[SELECT76:%.*]] = select i1 [[OR73]], <4 x float> zeroinitializer, <4 x float> [[SELECT75]]
+; IR-NEXT: [[SELECT77:%.*]] = select i1 [[ARG16]], <4 x float> zeroinitializer, <4 x float> [[SELECT76]]
+; IR-NEXT: [[ICMP78:%.*]] = icmp sge i32 [[ARG4]], [[CALL41]]
+; IR-NEXT: [[OR79:%.*]] = or i1 [[ARG24]], [[ICMP78]]
+; IR-NEXT: [[SELECT80:%.*]] = select i1 [[OR79]], <4 x float> zeroinitializer, <4 x float> [[ARG36]]
+; IR-NEXT: [[SELECT81:%.*]] = select i1 [[ARG5]], <4 x float> zeroinitializer, <4 x float> [[SELECT80]]
+; IR-NEXT: [[SELECT82:%.*]] = select i1 [[ARG6]], <4 x float> zeroinitializer, <4 x float> [[SELECT81]]
+; IR-NEXT: [[OR83:%.*]] = or i32 [[ARG1]], [[CALL]]
+; IR-NEXT: [[ICMP84:%.*]] = icmp sge i32 [[OR83]], 0
+; IR-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> [[SELECT82]], <4 x float> [[ARG37]]
+; IR-NEXT: br label %[[BB86]]
+; IR: [[BB86]]:
+; IR-NEXT: [[PHI87:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT46]], %[[BB40]] ]
+; IR-NEXT: [[PHI88:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT56]], %[[BB40]] ]
+; IR-NEXT: [[PHI89:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT63]], %[[BB40]] ]
+; IR-NEXT: [[PHI90:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT70]], %[[BB40]] ]
+; IR-NEXT: [[PHI91:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SELECT77]], %[[BB40]] ]
+; IR-NEXT: [[PHI92:%.*]] = phi <4 x float> [ zeroinitializer, %[[BB38]] ], [ [[SPEC_SELECT]], %[[BB40]] ]
+; IR-NEXT: [[EXTRACTELEMENT:%.*]] = extractelement <4 x float> [[PHI87]], i64 0
+; IR-NEXT: [[EXTRACTELEMENT93:%.*]] = extractelement <4 x float> [[PHI88]], i64 0
+; IR-NEXT: [[FMUL:%.*]] = fmul float [[EXTRACTELEMENT]], [[EXTRACTELEMENT93]]
+; IR-NEXT: [[EXTRACTELEMENT94:%.*]] = extractelement <4 x float> [[PHI89]], i64 0
+; IR-NEXT: [[FMUL95:%.*]] = fmul float [[FMUL]], [[EXTRACTELEMENT94]]
+; IR-NEXT: [[EXTRACTELEMENT96:%.*]] = extractelement <4 x float> [[PHI90]], i64 0
+; IR-NEXT: [[FMUL97:%.*]] = fmul float [[FMUL95]], [[EXTRACTELEMENT96]]
+; IR-NEXT: [[EXTRACTELEMENT98:%.*]] = extractelement <4 x float> [[PHI91]], i64 0
+; IR-NEXT: [[FMUL99:%.*]] = fmul float [[FMUL97]], [[EXTRACTELEMENT98]]
+; IR-NEXT: [[EXTRACTELEMENT100:%.*]] = extractelement <4 x float> [[PHI92]], i64 0
+; IR-NEXT: [[FMUL101:%.*]] = fmul float [[FMUL99]], [[EXTRACTELEMENT100]]
+; IR-NEXT: [[FMUL102]] = fmul float [[PHI]], [[FMUL101]]
+; IR-NEXT: [[EXTRACTELEMENT103:%.*]] = extractelement <4 x float> [[PHI39]], i64 0
+; IR-NEXT: [[FMUL104:%.*]] = fmul float [[EXTRACTELEMENT103]], 0.000000e+00
+; IR-NEXT: [[FPTRUNC:%.*]] = fptrunc float [[PHI]] to half
+; IR-NEXT: [[INSERTELEMENT:%.*]] = insertelement <4 x half> zeroinitializer, half [[FPTRUNC]], i64 0
+; IR-NEXT: [[CALL105]] = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> [[INSERTELEMENT]], <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB38]]
+;
+bb:
+ br label %bb38
+
+bb38:
+ %phi = phi float [ 0.000000e+00, %bb ], [ %fmul102, %bb86 ]
+ %phi39 = phi <4 x float> [ zeroinitializer, %bb ], [ %call105, %bb86 ]
+ br i1 %arg3, label %bb40, label %bb86
+
+bb40:
+ %call = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 0, i32 0)
+ %icmp = icmp sge i32 %arg9, %call
+ %or = or i1 %arg10, %icmp
+ %call41 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
+ %icmp42 = icmp sge i32 %arg18, %call41
+ %or43 = or i1 %arg19, %icmp42
+ %select = select i1 %or43, i1 false, i1 %arg3
+ %select44 = select i1 %or, <4 x float> zeroinitializer, <4 x float> %arg11
+ %select45 = select i1 %arg14, <4 x float> zeroinitializer, <4 x float> %select44
+ %select46 = select i1 %select, <4 x float> zeroinitializer, <4 x float> %select45
+ %icmp47 = icmp sle i32 %arg21, %call
+ %or48 = or i1 %icmp47, %arg22
+ %select49 = select i1 %or48, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %icmp50 = icmp sge i32 %arg23, %call41
+ %icmp51 = icmp sle i32 %arg13, %call41
+ %or52 = or i1 %icmp51, %arg29
+ %or53 = or i1 %arg8, %icmp50
+ %select54 = select i1 %arg25, <4 x float> zeroinitializer, <4 x float> %select49
+ %select55 = select i1 %or53, <4 x float> zeroinitializer, <4 x float> %select54
+ %select56 = select i1 %or52, <4 x float> zeroinitializer, <4 x float> %select55
+ %icmp57 = icmp sge i32 %arg30, %call
+ %or58 = or i1 %arg6, %icmp57
+ %icmp59 = icmp sle i32 %arg34, %call41
+ %or60 = or i1 %icmp59, %arg15
+ %select61 = select i1 %or58, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select62 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select61
+ %select63 = select i1 %or60, <4 x float> zeroinitializer, <4 x float> %select62
+ %icmp64 = icmp sle i32 %arg, %call
+ %icmp65 = icmp sge i32 %arg28, %call41
+ %or66 = or i1 %arg33, %icmp65
+ %or67 = or i1 %icmp64, %arg17
+ %select68 = select i1 %or67, <4 x float> zeroinitializer, <4 x float> %arg35
+ %select69 = select i1 %or66, <4 x float> zeroinitializer, <4 x float> %select68
+ %select70 = select i1 %arg7, <4 x float> zeroinitializer, <4 x float> %select69
+ %icmp71 = icmp sle i32 %arg32, %call
+ %icmp72 = icmp sge i32 %arg2, %call41
+ %or73 = or i1 %arg20, %icmp72
+ %or74 = or i1 %icmp71, %arg26
+ %select75 = select i1 %or74, <4 x float> splat (float 1.000000e+00), <4 x float> zeroinitializer
+ %select76 = select i1 %or73, <4 x float> zeroinitializer, <4 x float> %select75
+ %select77 = select i1 %arg16, <4 x float> zeroinitializer, <4 x float> %select76
+ %icmp78 = icmp sge i32 %arg4, %call41
+ %or79 = or i1 %arg24, %icmp78
+ %select80 = select i1 %or79, <4 x float> zeroinitializer, <4 x float> %arg36
+ %select81 = select i1 %arg5, <4 x float> zeroinitializer, <4 x float> %select80
+ %select82 = select i1 %arg6, <4 x float> zeroinitializer, <4 x float> %select81
+ %or83 = or i32 %arg1, %call
+ %icmp84 = icmp sge i32 %or83, 0
+ br i1 %icmp84, label %bb85, label %bb86
+
+bb85:
+ br label %bb86
+
+bb86:
+ %phi87 = phi <4 x float> [ %select46, %bb85 ], [ %select46, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi88 = phi <4 x float> [ %select56, %bb85 ], [ %select56, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi89 = phi <4 x float> [ %select63, %bb85 ], [ %select63, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi90 = phi <4 x float> [ %select70, %bb85 ], [ %select70, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi91 = phi <4 x float> [ %select77, %bb85 ], [ %select77, %bb40 ], [ zeroinitializer, %bb38 ]
+ %phi92 = phi <4 x float> [ %select82, %bb85 ], [ %arg37, %bb40 ], [ zeroinitializer, %bb38 ]
+ %extractelement = extractelement <4 x float> %phi87, i64 0
+ %extractelement93 = extractelement <4 x float> %phi88, i64 0
+ %fmul = fmul float %extractelement, %extractelement93
+ %extractelement94 = extractelement <4 x float> %phi89, i64 0
+ %fmul95 = fmul float %fmul, %extractelement94
+ %extractelement96 = extractelement <4 x float> %phi90, i64 0
+ %fmul97 = fmul float %fmul95, %extractelement96
+ %extractelement98 = extractelement <4 x float> %phi91, i64 0
+ %fmul99 = fmul float %fmul97, %extractelement98
+ %extractelement100 = extractelement <4 x float> %phi92, i64 0
+ %fmul101 = fmul float %fmul99, %extractelement100
+ %fmul102 = fmul float %phi, %fmul101
+ %extractelement103 = extractelement <4 x float> %phi39, i64 0
+ %fmul104 = fmul float %extractelement103, 0.000000e+00
+ %fptrunc = fptrunc float %phi to half
+ %insertelement = insertelement <4 x half> zeroinitializer, half %fptrunc, i64 0
+ %call105 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> %insertelement, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ br label %bb38
+}
+
+declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half>, <4 x half>, <4 x float>, i32, i32, i32)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
new file mode 100644
index 0000000000000..b1d4295fe7311
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/speculate-block.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=simplifycfg -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
+
+; Test that SimplifyCFG speculatively executes blocks on AMDGPU even for vector
+; types. The AMDGPU getCmpSelInstrCost override returns a low cost for selects,
+; allowing the speculation regardless of vector width.
+
+define amdgpu_kernel void @ham(i1 %arg3, i1 %icmp84) {
+; CHECK-LABEL: define amdgpu_kernel void @ham(
+; CHECK-SAME: i1 [[ARG3:%.*]], i1 [[ICMP84:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[ICMP84]], <4 x float> zeroinitializer, <4 x float> splat (float 1.000000e+00)
+; CHECK-NEXT: [[SPEC_SELECT1:%.*]] = select i1 [[ARG3]], <4 x float> [[SPEC_SELECT]], <4 x float> zeroinitializer
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %arg3, label %then, label %merge
+
+then: ; preds = %entry
+ %icmp841 = icmp sge i32 0, 0
+ br i1 %icmp84, label %then.if, label %merge
+
+then.if: ; preds = %then
+ br label %merge
+
+merge: ; preds = %then.if, %then, %entry
+ %phi92 = phi <4 x float> [ zeroinitializer, %then.if ], [ splat (float 1.000000e+00), %then ], [ zeroinitializer, %entry ]
+ ret void
+}
More information about the llvm-commits
mailing list