[llvm] 0a57fee - AMDGPU/GlobalISel: Legalize BF16 for FP math opcodes (#214059)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 08:59:52 PDT 2026
Author: vangthao95
Date: 2026-08-07T08:59:45-07:00
New Revision: 0a57fee7dc03ac3a958b416c0a778816f7cd36f4
URL: https://github.com/llvm/llvm-project/commit/0a57fee7dc03ac3a958b416c0a778816f7cd36f4
DIFF: https://github.com/llvm/llvm-project/commit/0a57fee7dc03ac3a958b416c0a778816f7cd36f4.diff
LOG: AMDGPU/GlobalISel: Legalize BF16 for FP math opcodes (#214059)
Make the BF16 form of G_FCONSTANT legal.
Widen BF16 G_FADD, G_FMUL, G_FMA, and G_FCANONICALIZE to F32.
Added:
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 958aff4e1b22d..0df816eb93701 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -939,9 +939,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.clampScalar(0, S32, S64)
.widenScalarToNextPow2(0);
- getActionDefinitionsBuilder(G_FCONSTANT)
- .legalFor({S32, S64, S16})
- .clampScalar(0, S16, S64);
+ getActionDefinitionsBuilder(G_FCONSTANT).legalFor({F32, F64, F16, BF16});
getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
.legalIf(isRegisterClassType(ST, 0))
@@ -976,36 +974,50 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
auto &FPOpActions =
- getActionDefinitionsBuilder({G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
- G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
+ getActionDefinitionsBuilder({G_FADD, G_FMUL, G_FMA}).legalFor({F32, F64});
+ auto &FCanonicalizeActions =
+ getActionDefinitionsBuilder(G_FCANONICALIZE).legalFor({F32, F64});
+ auto &StrictFPOpActions =
+ getActionDefinitionsBuilder({G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
.legalFor({F32, F64});
auto &TrigActions =
getActionDefinitionsBuilder({G_FSIN, G_FCOS}).customFor({F32, F64});
auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV).customFor({F32, F64});
if (ST.has16BitInsts()) {
- if (ST.hasVOP3PInsts())
+ if (ST.hasVOP3PInsts()) {
FPOpActions.legalFor({F16, V2F16});
- else
+ FCanonicalizeActions.legalFor({F16, V2F16});
+ StrictFPOpActions.legalFor({F16, V2F16});
+ } else {
FPOpActions.legalFor({F16});
+ FCanonicalizeActions.legalFor({F16});
+ StrictFPOpActions.legalFor({F16});
+ }
TrigActions.customFor({F16});
FDIVActions.customFor({F16});
}
+ FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
+ FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
+
if (ST.hasAnyPackedFP32Ops()) {
FPOpActions.legalFor({V2F32});
+ FCanonicalizeActions.legalFor({V2F32});
+ StrictFPOpActions.legalFor({V2F32});
FPOpActions.clampMaxNumElementsStrict(0, F32, 2);
+ FCanonicalizeActions.clampMaxNumElementsStrict(0, F32, 2);
+ StrictFPOpActions.clampMaxNumElementsStrict(0, F32, 2);
}
if (ST.hasAnyPackedFP64Ops()) {
FPOpActions.legalFor({V2F64});
+ FCanonicalizeActions.legalFor({V2F64});
+ StrictFPOpActions.legalFor({V2F64});
FPOpActions.clampMaxNumElementsStrict(0, F64, 2);
- }
-
- if (ST.hasAnyPackedFP64Ops()) {
- FPOpActions.legalFor({V2F64});
- FPOpActions.clampMaxNumElementsStrict(0, F64, 2);
+ FCanonicalizeActions.clampMaxNumElementsStrict(0, F64, 2);
+ StrictFPOpActions.clampMaxNumElementsStrict(0, F64, 2);
}
auto &MinNumMaxNumIeee =
@@ -1047,16 +1059,23 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
MinNumMaxNum.minScalar(0, F32);
}
- if (ST.hasVOP3PInsts())
+ if (ST.hasVOP3PInsts()) {
FPOpActions.clampMaxNumElementsStrict(0, F16, 2);
+ FCanonicalizeActions.clampMaxNumElementsStrict(0, F16, 2);
+ StrictFPOpActions.clampMaxNumElementsStrict(0, F16, 2);
+ }
FPOpActions.scalarize(0);
+ FCanonicalizeActions.scalarize(0);
+ StrictFPOpActions.scalarize(0);
TrigActions.scalarize(0);
FDIVActions.scalarize(0);
if (!ST.has16BitInsts()) {
- FPOpActions.minScalar(0, F32);
- TrigActions.minScalar(0, F32);
- FDIVActions.minScalar(0, F32);
+ FPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
+ FCanonicalizeActions.widenScalarFor({F16}, changeElementTo(0, F32));
+ StrictFPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
+ TrigActions.widenScalarFor({F16}, changeElementTo(0, F32));
+ FDIVActions.widenScalarFor({F16}, changeElementTo(0, F32));
}
auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
new file mode 100644
index 0000000000000..a32a36cb78e77
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -0,0 +1,497 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fadd_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fadd_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fadd_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_add_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_add_f32 s0, s0, s1
+; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fadd_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1250-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX1250-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_add_f32 v2, v2, v3
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fadd_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_add_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_add_f32 v0, s0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_add_f32_e32 v1, s1, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: s_lshr_b32 s1, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_add_f32 v0, s0, v0 :: v_dual_add_f32 v1, s1, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fadd_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fadd_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_add_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_add_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_add_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s3, s1, 16
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_add_f32 s2, s2, s3
+; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX1250-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s3, s3, s2
+; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s2, 0
+; GFX1250-NEXT: s_cselect_b32 s2, s4, s3
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
+; GFX1250-NEXT: s_add_f32 s0, s0, s1
+; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s3, s1
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fadd_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_add_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_add_f32 v0, 2.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_add_f32_e32 v1, 2.0, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_add_f32 v0, 2.0, v0 :: v_dual_add_f32 v1, 2.0, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fadd_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_add_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_add_f32 v0, 1.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_add_f32_e32 v1, 0x42c80000, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v1, 0x42c80000, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
+ ret <2 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
new file mode 100644
index 0000000000000..9056a1b641e1c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fcanonicalize_bf16_v(bfloat %src) {
+; GFX9-LABEL: fcanonicalize_bf16_v:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_bf16_v:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_bf16_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
+; GFX9-LABEL: fcanonicalize_bf16_s:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_max_f32_e64 v0, s0, s0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_bf16_s:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_bf16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_v(<2 x bfloat> %src) {
+; GFX9-LABEL: fcanonicalize_v2bf16_v:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_v2bf16_v:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_v2bf16_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_max_num_f32 v1, v1, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat> %src)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
+; GFX9-LABEL: fcanonicalize_v2bf16_s:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_max_f32_e64 v0, s0, s0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s3, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s3, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: v_max_f32_e64 v0, s1, s1
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_v2bf16_s:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_max_num_f32_e64 v0, s1, s1
+; GFX12-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12-NEXT: s_bfe_u32 s2, s1, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s1, 0x400000
+; GFX12-NEXT: s_add_co_i32 s2, s2, s1
+; GFX12-NEXT: s_addk_co_i32 s2, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s1, 0
+; GFX12-NEXT: s_cselect_b32 s1, s3, s2
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s2, s2, s0
+; GFX12-NEXT: s_addk_co_i32 s2, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_v2bf16_s:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s1, s0, 16
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_max_num_f32_e64 v0, s1, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v0
+; GFX1250-NEXT: s_bfe_u32 s2, s1, 0x10010
+; GFX1250-NEXT: s_or_b32 s3, s1, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s2, s2, s1
+; GFX1250-NEXT: s_addk_co_i32 s2, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s1, 0
+; GFX1250-NEXT: s_cselect_b32 s1, s3, s2
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s2, s2, s0
+; GFX1250-NEXT: s_addk_co_i32 s2, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s3, s2
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat> %src)
+ ret <2 x bfloat> %result
+}
+
+declare bfloat @llvm.canonicalize.bf16(bfloat)
+declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
new file mode 100644
index 0000000000000..015c0289d7712
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -0,0 +1,555 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fma_bf16_vvv(bfloat %a, bfloat %b, bfloat %c) {
+; GFX9-LABEL: fma_bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX12-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX1250-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX1250-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat inreg %c) {
+; GFX9-LABEL: fma_bf16_sss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_fma_f32 v0, s0, v0, v1
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_bf16_sss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshl_b32 s2, s2, 16
+; GFX12-NEXT: s_fmac_f32 s2, s0, s1
+; GFX12-NEXT: s_bfe_u32 s0, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s1, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s0, s0, s2
+; GFX12-NEXT: s_addk_co_i32 s0, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s0, s1, s0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_bf16_sss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
+; GFX1250-NEXT: s_fmac_f32 s2, s0, s1
+; GFX1250-NEXT: s_bfe_u32 s0, s2, 0x10010
+; GFX1250-NEXT: s_or_b32 s1, s2, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
+; GFX1250-NEXT: s_addk_co_i32 s0, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s2, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fma_v2bf16_vvv(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; GFX9-LABEL: fma_v2bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v6, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_fma_f32 v1, v1, v2, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v2bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_dual_fmac_f32 v2, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_fmac_f32_e32 v5, v3, v4
+; GFX12-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX12-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX12-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v2bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v3.l, v0.h
+; GFX1250-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX1250-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX1250-NEXT: v_dual_lshlrev_b32 v3, 16, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX1250-NEXT: v_dual_lshlrev_b32 v5, 16, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_dual_fmac_f32 v5, v3, v4 :: v_dual_fmac_f32 v2, v0, v1
+; GFX1250-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX1250-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v3, 0x400000, v5
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX1250-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 16, v0 :: v_dual_lshrrev_b32 v0, 16, v1
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fma_v2bf16_vss(<2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
+; GFX9-LABEL: fma_v2bf16_vss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_fma_f32 v0, v0, s0, v2
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_lshl_b32 s1, s3, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s2, 16
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_fma_f32 v1, v1, s0, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v2bf16_vss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-NEXT: s_lshr_b32 s3, s1, 16
+; GFX12-NEXT: s_lshl_b32 s2, s2, 16
+; GFX12-NEXT: s_lshl_b32 s3, s3, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: v_fma_f32 v1, v1, s2, s3
+; GFX12-NEXT: v_fma_f32 v0, v0, s0, s1
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v2bf16_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: s_lshr_b32 s2, s0, 16
+; GFX1250-NEXT: s_lshr_b32 s3, s1, 16
+; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
+; GFX1250-NEXT: s_lshl_b32 s3, s3, 16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: v_fma_f32 v1, v1, s2, s3
+; GFX1250-NEXT: v_fma_f32 v0, v0, s0, s1
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fma_v2bf16_sss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
+; GFX9-LABEL: fma_v2bf16_sss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_fma_f32 v0, s0, v0, v1
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s3, 16
+; GFX9-NEXT: s_lshl_b32 s2, s4, 16
+; GFX9-NEXT: s_lshl_b32 s3, s5, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_fma_f32 v0, s1, v0, v1
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v2bf16_sss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s3, s0, 16
+; GFX12-NEXT: s_lshl_b32 s4, s1, 16
+; GFX12-NEXT: s_lshl_b32 s5, s2, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_fmac_f32 s5, s3, s4
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_bfe_u32 s3, s5, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s5, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s5
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s5, 0
+; GFX12-NEXT: s_cselect_b32 s3, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshl_b32 s2, s2, 16
+; GFX12-NEXT: s_fmac_f32 s2, s0, s1
+; GFX12-NEXT: s_lshr_b32 s1, s3, 16
+; GFX12-NEXT: s_bfe_u32 s0, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s0, s0, s2
+; GFX12-NEXT: s_addk_co_i32 s0, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s0
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v2bf16_sss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s3, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s4, s1, 16
+; GFX1250-NEXT: s_lshl_b32 s5, s2, 16
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_fmac_f32 s5, s3, s4
+; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
+; GFX1250-NEXT: s_bfe_u32 s3, s5, 0x10010
+; GFX1250-NEXT: s_or_b32 s4, s5, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s3, s3, s5
+; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s5, 0
+; GFX1250-NEXT: s_cselect_b32 s3, s4, s3
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
+; GFX1250-NEXT: s_fmac_f32 s2, s0, s1
+; GFX1250-NEXT: s_lshr_b32 s1, s3, 16
+; GFX1250-NEXT: s_bfe_u32 s0, s2, 0x10010
+; GFX1250-NEXT: s_or_b32 s3, s2, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
+; GFX1250-NEXT: s_addk_co_i32 s0, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s2, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s3, s0
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fma_v2bf16_vsc(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fma_v2bf16_vsc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_fma_f32 v0, v0, s0, 0.5
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_fma_f32 v1, v1, s0, 0.5
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v2bf16_vsc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_fma_f32 v0, v0, s0, 0.5
+; GFX12-NEXT: v_fma_f32 v1, v1, s1, 0.5
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v2bf16_vsc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: s_lshr_b32 s1, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_fma_f32 v0, v0, s0, 0.5
+; GFX1250-NEXT: v_fma_f32 v1, v1, s1, 0.5
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
+; GFX9-LABEL: fma_v2bf16_vll:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_fma_f32 v0, v0, 1.0, 2.0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x42c80000
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x43480000
+; GFX9-NEXT: v_fma_f32 v1, v1, v2, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v2bf16_vll:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_mov_b32 v2, 0x43480000 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_fma_f32 v0, v0, 1.0, 2.0
+; GFX12-NEXT: v_fmac_f32_e32 v2, 0x42c80000, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v2bf16_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0x43480000 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_fmac_f32_e32 v2, 0x42c80000, v1
+; GFX1250-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshrrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_fma_f32 v0, v0, 1.0, 2.0
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, <2 x bfloat> <bfloat 2.0, bfloat 200.0>)
+ ret <2 x bfloat> %result
+}
+
+declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
+declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
new file mode 100644
index 0000000000000..bd22b6a862a69
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -0,0 +1,497 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fmul_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fmul_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fmul_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_mul_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_mul_f32 s0, s0, s1
+; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fmul_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_mul_f32_e32 v2, v2, v3
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1250-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX1250-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_mul_f32 v2, v2, v3
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmul_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshr_b32 s1, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_mul_f32 v0, s0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_mul_f32_e32 v1, s1, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: s_lshr_b32 s1, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_mul_f32 v0, s0, v0 :: v_dual_mul_f32 v1, s1, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmul_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmul_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mul_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_mul_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_mul_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s3, s1, 16
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_mul_f32 s2, s2, s3
+; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX1250-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s3, s3, s2
+; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s2, 0
+; GFX1250-NEXT: s_cselect_b32 s2, s4, s3
+; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
+; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
+; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
+; GFX1250-NEXT: s_mul_f32 s0, s0, s1
+; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-NEXT: s_cselect_b32 s0, s3, s1
+; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fmul_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0.5, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0.5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_mul_f32 v0, 0.5, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_mul_f32_e32 v1, 0.5, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_mul_f32 v0, 0.5, v0 :: v_dual_mul_f32 v1, 0.5, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <2 x bfloat> %a, <bfloat 0.5, bfloat 0.5>
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fmul_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_mul_f32 v0, 1.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX12-NEXT: v_mul_f32_e32 v1, 0x42c80000, v1
+; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_dual_mul_f32 v0, 1.0, v0 :: v_dual_mul_f32 v1, 0x42c80000, v1
+; GFX1250-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1250-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1250-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 16, v1 :: v_dual_lshrrev_b32 v0, 16, v0
+; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
+ ret <2 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 82313437412a9..dd8fa98c62afc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -1,9 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
-# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GFX9 %s
+# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX10 %s
+# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX1250 %s
---
name: test_fadd_f32
@@ -27,13 +28,13 @@ body: |
; VI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[COPY]], [[COPY1]]
; VI-NEXT: $vgpr0 = COPY [[FADD]](f32)
;
- ; GFX9-LABEL: name: test_fadd_f32
- ; GFX9: liveins: $vgpr0, $vgpr1
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[COPY]], [[COPY1]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FADD]](f32)
+ ; GCN-LABEL: name: test_fadd_f32
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FADD]](f32)
%0:_(f32) = COPY $vgpr0
%1:_(f32) = COPY $vgpr1
%2:_(f32) = G_FADD %0, %1
@@ -62,13 +63,13 @@ body: |
; VI-NEXT: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[COPY]], [[COPY1]]
; VI-NEXT: $vgpr0_vgpr1 = COPY [[FADD]](f64)
;
- ; GFX9-LABEL: name: test_fadd_f64
- ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[COPY]], [[COPY1]]
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FADD]](f64)
+ ; GCN-LABEL: name: test_fadd_f64
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[FADD]](f64)
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f64) = COPY $vgpr2_vgpr3
%2:_(f64) = G_FADD %0, %1
@@ -106,16 +107,16 @@ body: |
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FADD]](f16)
; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
- ; GFX9-LABEL: name: test_fadd_f16
- ; GFX9: liveins: $vgpr0, $vgpr1
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f16) = G_FADD [[TRUNC]], [[TRUNC1]]
- ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FADD]](f16)
- ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; GCN-LABEL: name: test_fadd_f16
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GCN-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
+ ; GCN-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(f16) = G_FADD [[TRUNC]], [[TRUNC1]]
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FADD]](f16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
%0:_(i32) = COPY $vgpr0
%1:_(i32) = COPY $vgpr1
%2:_(f16) = G_TRUNC %0
@@ -167,6 +168,38 @@ body: |
; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[UV1]], [[UV3]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v2f32
+ ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX10-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[UV]], [[UV2]]
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[UV1]], [[UV3]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v2f32
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[UV]], [[UV2]]
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[UV1]], [[UV3]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v2f32
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x f32>) = G_FADD [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FADD]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = COPY $vgpr2_vgpr3
%2:_(<2 x f32>) = G_FADD %0, %1
@@ -214,6 +247,38 @@ body: |
; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(f32) = nnan G_FADD [[UV1]], [[UV3]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v2f32_flags
+ ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX10-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = nnan G_FADD [[UV]], [[UV2]]
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(f32) = nnan G_FADD [[UV1]], [[UV3]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v2f32_flags
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = nnan G_FADD [[UV]], [[UV2]]
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(f32) = nnan G_FADD [[UV1]], [[UV3]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v2f32_flags
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x f32>) = nnan G_FADD [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FADD]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = COPY $vgpr2_vgpr3
%2:_(<2 x f32>) = nnan G_FADD %0, %1
@@ -264,6 +329,51 @@ body: |
; GFX9-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[UV2]], [[UV5]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32), [[FADD2]](f32)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v3f32
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX10-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[UV]], [[UV3]]
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[UV1]], [[UV4]]
+ ; GFX10-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[UV2]], [[UV5]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32), [[FADD2]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v3f32
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[UV]], [[UV3]]
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[UV1]], [[UV4]]
+ ; GFX11-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[UV2]], [[UV5]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FADD]](f32), [[FADD1]](f32), [[FADD2]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v3f32
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV]](f32), [[UV1]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV2]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV3]](f32), [[UV4]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV5]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x f32>) = G_FADD [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+ ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f32>) = G_FADD [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FADD]](<2 x f32>)
+ ; GFX1250-NEXT: [[UV8:%[0-9]+]]:_(f32), [[UV9:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FADD1]](<2 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[UV6]](f32), [[UV7]](f32), [[UV8]](f32)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR4]](<3 x f32>)
%0:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<3 x f32>) = G_FADD %0, %1
@@ -300,17 +410,17 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FADD]](f64), [[FADD1]](f64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
;
- ; GFX9-LABEL: name: test_fadd_v2f64
- ; GFX9: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[UV]], [[UV2]]
- ; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(f64) = G_FADD [[UV1]], [[UV3]]
- ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FADD]](f64), [[FADD1]](f64)
- ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
+ ; GCN-LABEL: name: test_fadd_v2f64
+ ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[UV]], [[UV2]]
+ ; GCN-NEXT: [[FADD1:%[0-9]+]]:_(f64) = G_FADD [[UV1]], [[UV3]]
+ ; GCN-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FADD]](f64), [[FADD1]](f64)
+ ; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
%2:_(<2 x f64>) = G_FADD %0, %1
@@ -375,13 +485,13 @@ body: |
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
;
- ; GFX9-LABEL: name: test_fadd_v2f16
- ; GFX9: liveins: $vgpr0, $vgpr1
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[COPY]], [[COPY1]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FADD]](<2 x f16>)
+ ; GCN-LABEL: name: test_fadd_v2f16
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FADD]](<2 x f16>)
%0:_(<2 x f16>) = COPY $vgpr0
%1:_(<2 x f16>) = COPY $vgpr1
%2:_(<2 x f16>) = G_FADD %0, %1
@@ -500,6 +610,70 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v3f16
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV]], [[UV3]]
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV1]], [[UV4]]
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](<2 x f16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](<2 x f16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v3f16
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV]], [[UV3]]
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV1]], [[UV4]]
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v3f16
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16), [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(<2 x f16>), [[UV7:%[0-9]+]]:_(<2 x f16>), [[UV8:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV2]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[UV9:%[0-9]+]]:_(f16), [[UV10:%[0-9]+]]:_(f16), [[UV11:%[0-9]+]]:_(f16), [[UV12:%[0-9]+]]:_(f16), [[UV13:%[0-9]+]]:_(f16), [[UV14:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[UV15:%[0-9]+]]:_(<2 x f16>), [[UV16:%[0-9]+]]:_(<2 x f16>), [[UV17:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV11]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV6]], [[UV15]]
+ ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f16>) = G_FADD [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[FADD]](<2 x f16>), [[FADD1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
%0:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<3 x f16>), %3:_(<3 x f16>) = G_UNMERGE_VALUES %0
@@ -611,19 +785,1012 @@ body: |
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
;
- ; GFX9-LABEL: name: test_fadd_v4f16
- ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
- ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV]], [[UV2]]
- ; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV1]], [[UV3]]
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FADD]](<2 x f16>), [[FADD1]](<2 x f16>)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
+ ; GCN-LABEL: name: test_fadd_v4f16
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
+ ; GCN-NEXT: [[FADD:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV]], [[UV2]]
+ ; GCN-NEXT: [[FADD1:%[0-9]+]]:_(<2 x f16>) = G_FADD [[UV1]], [[UV3]]
+ ; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FADD]](<2 x f16>), [[FADD1]](<2 x f16>)
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
%0:_(<4 x f16>) = COPY $vgpr0_vgpr1
%1:_(<4 x f16>) = COPY $vgpr2_vgpr3
%2:_(<4 x f16>) = G_FADD %0, %1
$vgpr0_vgpr1 = COPY %2
...
+
+---
+name: test_fadd_bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fadd_bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST]], [[BITCAST1]]
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; VI-LABEL: name: test_fadd_bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST]], [[BITCAST1]]
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX9-LABEL: name: test_fadd_bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST]], [[BITCAST1]]
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX10-LABEL: name: test_fadd_bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST]], [[BITCAST1]]
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX11-LABEL: name: test_fadd_bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST]], [[BITCAST1]]
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ %0:_(bf16) = G_IMPLICIT_DEF
+ %1:_(bf16) = G_IMPLICIT_DEF
+ %2:_(bf16) = G_FADD %0, %1
+ S_NOP 0, implicit %2
+...
+
+---
+name: test_fadd_v2bf16
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ ; SI-LABEL: name: test_fadd_v2bf16
+ ; SI: liveins: $vgpr0, $vgpr1
+ ; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x bf16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x bf16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST2]], [[BITCAST3]]
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[BITCAST6]]
+ ; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C]](i32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL4]]
+ ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; SI-NEXT: $vgpr0 = COPY [[BITCAST8]](<2 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fadd_v2bf16
+ ; VI: liveins: $vgpr0, $vgpr1
+ ; VI-NEXT: {{ $}}
+ ; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x bf16>)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x bf16>)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST2]], [[BITCAST3]]
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[BITCAST6]]
+ ; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL4]]
+ ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; VI-NEXT: $vgpr0 = COPY [[BITCAST8]](<2 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fadd_v2bf16
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x bf16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x bf16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST2]], [[BITCAST3]]
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[BITCAST6]]
+ ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX9-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v2bf16
+ ; GFX10: liveins: $vgpr0, $vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x bf16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x bf16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST2]], [[BITCAST3]]
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[BITCAST6]]
+ ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX10-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v2bf16
+ ; GFX11: liveins: $vgpr0, $vgpr1
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x bf16>)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x bf16>)
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST2]], [[BITCAST3]]
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[BITCAST6]]
+ ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX11-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v2bf16
+ ; GFX1250: liveins: $vgpr0, $vgpr1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY]](<2 x bf16>)
+ ; GFX1250-NEXT: [[UV2:%[0-9]+]]:_(bf16), [[UV3:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY1]](<2 x bf16>)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV2]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[UV1]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[UV3]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST9]]
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
+ ; GFX1250-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x bf16>)
+ %0:_(<2 x bf16>) = COPY $vgpr0
+ %1:_(<2 x bf16>) = COPY $vgpr1
+ %2:_(<2 x bf16>) = G_FADD %0, %1
+ $vgpr0 = COPY %2
+...
+
+---
+name: test_fadd_v4bf16
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+ ; SI-LABEL: name: test_fadd_v4bf16
+ ; SI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x bf16>)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x bf16>)
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x bf16>)
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x bf16>)
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST4]], [[BITCAST5]]
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST8]]
+ ; SI-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[C1]]
+ ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST9]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; SI-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST10]], [[BITCAST11]]
+ ; SI-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST12]], [[C1]]
+ ; SI-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C2]]
+ ; SI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; SI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST12]], [[ADD4]]
+ ; SI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; SI-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; SI-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST14]]
+ ; SI-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; SI-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR10]], [[C2]]
+ ; SI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; SI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; SI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; SI-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL8]]
+ ; SI-NEXT: [[BITCAST16:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR11]], [[C]](i32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR9]], [[SHL9]]
+ ; SI-NEXT: [[BITCAST17:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST16]](<2 x bf16>), [[BITCAST17]](<2 x bf16>)
+ ; SI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fadd_v4bf16
+ ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; VI-NEXT: {{ $}}
+ ; VI-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x bf16>)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x bf16>)
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x bf16>)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x bf16>)
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST4]], [[BITCAST5]]
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST8]]
+ ; VI-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[C1]]
+ ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST9]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; VI-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST10]], [[BITCAST11]]
+ ; VI-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST12]], [[C1]]
+ ; VI-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C2]]
+ ; VI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; VI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST12]], [[ADD4]]
+ ; VI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; VI-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; VI-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST14]]
+ ; VI-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; VI-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR10]], [[C2]]
+ ; VI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; VI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; VI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; VI-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL8]]
+ ; VI-NEXT: [[BITCAST16:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR11]], [[C]](i32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR9]], [[SHL9]]
+ ; VI-NEXT: [[BITCAST17:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST16]](<2 x bf16>), [[BITCAST17]](<2 x bf16>)
+ ; VI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fadd_v4bf16
+ ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x bf16>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x bf16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x bf16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x bf16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST4]], [[BITCAST5]]
+ ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX9-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST8]]
+ ; GFX9-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[C1]]
+ ; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST9]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX9-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST10]], [[BITCAST11]]
+ ; GFX9-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST12]], [[C1]]
+ ; GFX9-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C2]]
+ ; GFX9-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX9-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST12]], [[ADD4]]
+ ; GFX9-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX9-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR9]](i32)
+ ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX9-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST14]]
+ ; GFX9-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX9-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR10]], [[C2]]
+ ; GFX9-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX9-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX9-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX9-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR11]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX10-LABEL: name: test_fadd_v4bf16
+ ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x bf16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x bf16>)
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x bf16>)
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x bf16>)
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST2]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST4]], [[BITCAST5]]
+ ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX10-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST8]]
+ ; GFX10-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[C1]]
+ ; GFX10-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST9]], [[ADD2]]
+ ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX10-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX10-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST10]], [[BITCAST11]]
+ ; GFX10-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST12]], [[C1]]
+ ; GFX10-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; GFX10-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C2]]
+ ; GFX10-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX10-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST12]], [[ADD4]]
+ ; GFX10-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX10-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR9]](i32)
+ ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX10-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST14]]
+ ; GFX10-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX10-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX10-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR10]], [[C2]]
+ ; GFX10-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX10-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX10-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX10-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR11]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fadd_v4bf16
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x bf16>)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x bf16>)
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x bf16>)
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x bf16>)
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST4]], [[BITCAST5]]
+ ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX11-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST8]]
+ ; GFX11-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[C1]]
+ ; GFX11-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST9]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX11-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX11-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX11-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST10]], [[BITCAST11]]
+ ; GFX11-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; GFX11-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST12]], [[C1]]
+ ; GFX11-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; GFX11-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C2]]
+ ; GFX11-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX11-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST12]], [[ADD4]]
+ ; GFX11-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX11-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR9]](i32)
+ ; GFX11-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX11-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX11-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST14]]
+ ; GFX11-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; GFX11-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX11-NEXT: [[LSHR10:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX11-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR10]], [[C2]]
+ ; GFX11-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX11-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX11-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX11-NEXT: [[LSHR11:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR11]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fadd_v4bf16
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16), [[UV2:%[0-9]+]]:_(bf16), [[UV3:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:_(bf16), [[UV5:%[0-9]+]]:_(bf16), [[UV6:%[0-9]+]]:_(bf16), [[UV7:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[UV1]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST9]]
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[UV2]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
+ ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST14]](i16)
+ ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX1250-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST15]]
+ ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
+ ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST16]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST16]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST16]], [[ADD4]]
+ ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[UV3]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
+ ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[UV7]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
+ ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX1250-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST19]], [[BITCAST21]]
+ ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
+ ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD6]]
+ ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST17]](bf16), [[BITCAST23]](bf16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
+ %0:_(<4 x bf16>) = COPY $vgpr0_vgpr1
+ %1:_(<4 x bf16>) = COPY $vgpr2_vgpr3
+ %2:_(<4 x bf16>) = G_FADD %0, %1
+ $vgpr0_vgpr1 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index 27abadc18d923..60e24cdc230d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -1,9 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
-# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GFX9 %s
+# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX10 %s
+# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX1250 %s
---
@@ -26,12 +27,12 @@ body: |
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[COPY]]
; VI-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](f32)
;
- ; GFX9-LABEL: name: test_fcanonicalize_f32
- ; GFX9: liveins: $vgpr0
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[COPY]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](f32)
+ ; GCN-LABEL: name: test_fcanonicalize_f32
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
+ ; GCN-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[COPY]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](f32)
%0:_(f32) = COPY $vgpr0
%1:_(f32) = G_FCANONICALIZE %0
$vgpr0 = COPY %1
@@ -54,11 +55,11 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
; VI-NEXT: $vgpr0_vgpr1 = COPY [[COPY]](f64)
;
- ; GFX9-LABEL: name: test_fcanonicalize_f64
- ; GFX9: liveins: $vgpr0
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[COPY]](f64)
+ ; GCN-LABEL: name: test_fcanonicalize_f64
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[COPY]](f64)
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f64) = G_FCANONICALIZE %0
$vgpr0_vgpr1 = COPY %0
@@ -89,14 +90,14 @@ body: |
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FCANONICALIZE]](f16)
; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
- ; GFX9-LABEL: name: test_fcanonicalize_f16
- ; GFX9: liveins: $vgpr0
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
- ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f16) = G_FCANONICALIZE [[TRUNC]]
- ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FCANONICALIZE]](f16)
- ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; GCN-LABEL: name: test_fcanonicalize_f16
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GCN-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
+ ; GCN-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f16) = G_FCANONICALIZE [[TRUNC]]
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FCANONICALIZE]](f16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
%0:_(i32) = COPY $vgpr0
%1:_(f16) = G_TRUNC %0
%2:_(f16) = G_FCANONICALIZE %1
@@ -139,6 +140,33 @@ body: |
; GFX9-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV1]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fcanonicalize_v2f32
+ ; GFX10: liveins: $vgpr0_vgpr1
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX10-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV]]
+ ; GFX10-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV1]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fcanonicalize_v2f32
+ ; GFX11: liveins: $vgpr0_vgpr1
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV]]
+ ; GFX11-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV1]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fcanonicalize_v2f32
+ ; GFX1250: liveins: $vgpr0_vgpr1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f32>) = G_FCANONICALIZE [[COPY]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FCANONICALIZE]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = G_FCANONICALIZE %0
$vgpr0_vgpr1 = COPY %1
@@ -182,6 +210,43 @@ body: |
; GFX9-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV2]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32), [[FCANONICALIZE2]](f32)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fcanonicalize_v3f32
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX10-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV]]
+ ; GFX10-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV1]]
+ ; GFX10-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV2]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32), [[FCANONICALIZE2]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fcanonicalize_v3f32
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX11-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV]]
+ ; GFX11-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV1]]
+ ; GFX11-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[UV2]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FCANONICALIZE]](f32), [[FCANONICALIZE1]](f32), [[FCANONICALIZE2]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fcanonicalize_v3f32
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV]](f32), [[UV1]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV2]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f32>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX1250-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f32>) = G_FCANONICALIZE [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FCANONICALIZE]](<2 x f32>)
+ ; GFX1250-NEXT: [[UV5:%[0-9]+]]:_(f32), [[UV6:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FCANONICALIZE1]](<2 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[UV3]](f32), [[UV4]](f32), [[UV5]](f32)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR2]](<3 x f32>)
%0:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x f32>) = G_FCANONICALIZE %0
$vgpr0_vgpr1_vgpr2 = COPY %1
@@ -213,15 +278,15 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FCANONICALIZE]](f64), [[FCANONICALIZE1]](f64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
;
- ; GFX9-LABEL: name: test_fcanonicalize_v2f64
- ; GFX9: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
- ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f64) = G_FCANONICALIZE [[UV]]
- ; GFX9-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f64) = G_FCANONICALIZE [[UV1]]
- ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FCANONICALIZE]](f64), [[FCANONICALIZE1]](f64)
- ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
+ ; GCN-LABEL: name: test_fcanonicalize_v2f64
+ ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; GCN-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f64) = G_FCANONICALIZE [[UV]]
+ ; GCN-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f64) = G_FCANONICALIZE [[UV1]]
+ ; GCN-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FCANONICALIZE]](f64), [[FCANONICALIZE1]](f64)
+ ; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f64>) = G_FCANONICALIZE %0
$vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -273,12 +338,12 @@ body: |
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x f16>)
;
- ; GFX9-LABEL: name: test_fcanonicalize_v2f16
- ; GFX9: liveins: $vgpr0
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[COPY]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](<2 x f16>)
+ ; GCN-LABEL: name: test_fcanonicalize_v2f16
+ ; GCN: liveins: $vgpr0
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
+ ; GCN-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[COPY]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](<2 x f16>)
%0:_(<2 x f16>) = COPY $vgpr0
%1:_(<2 x f16>) = G_FCANONICALIZE %0
$vgpr0 = COPY %1
@@ -329,6 +394,46 @@ body: |
; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](<2 x f16>)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x i32>)
+ ;
+ ; GFX10-LABEL: name: test_fcanonicalize_v3f16
+ ; GFX10: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX10-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR1]]
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](<2 x f16>)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](<2 x f16>)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
+ ; GFX10-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x i32>)
+ ;
+ ; GFX11-LABEL: name: test_fcanonicalize_v3f16
+ ; GFX11: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX11-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR1]]
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](<2 x f16>)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](<2 x f16>)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](i32), [[LSHR]](i32), [[BITCAST1]](i32)
+ ; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x i32>)
+ ;
+ ; GFX1250-LABEL: name: test_fcanonicalize_v3f16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX1250-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[FCANONICALIZE]](<2 x f16>)
+ ; GFX1250-NEXT: [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[FCANONICALIZE1]](<2 x f16>)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV]](f16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV1]](f16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV2]](f16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR2]](<3 x i32>)
%0:_(<3 x f16>) = G_IMPLICIT_DEF
%1:_(<3 x f16>) = G_FCANONICALIZE %0
%2:_(<3 x i32>) = G_ANYEXT %1
@@ -411,16 +516,387 @@ body: |
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
;
- ; GFX9-LABEL: name: test_fcanonicalize_v4f16
- ; GFX9: liveins: $vgpr0_vgpr1
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
- ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[UV]]
- ; GFX9-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[UV1]]
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FCANONICALIZE]](<2 x f16>), [[FCANONICALIZE1]](<2 x f16>)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
+ ; GCN-LABEL: name: test_fcanonicalize_v4f16
+ ; GCN: liveins: $vgpr0_vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
+ ; GCN-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[UV]]
+ ; GCN-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x f16>) = G_FCANONICALIZE [[UV1]]
+ ; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FCANONICALIZE]](<2 x f16>), [[FCANONICALIZE1]](<2 x f16>)
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
%0:_(<4 x f16>) = COPY $vgpr0_vgpr1
%1:_(<4 x f16>) = G_FCANONICALIZE %0
$vgpr0_vgpr1 = COPY %1
...
+---
+name: test_fcanonicalize_bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fcanonicalize_bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; VI-LABEL: name: test_fcanonicalize_bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX9-LABEL: name: test_fcanonicalize_bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX10-LABEL: name: test_fcanonicalize_bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX11-LABEL: name: test_fcanonicalize_bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX1250-LABEL: name: test_fcanonicalize_bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST1]]
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST3]](bf16)
+ %0:_(bf16) = G_IMPLICIT_DEF
+ %1:_(bf16) = G_FCANONICALIZE %0
+ S_NOP 0, implicit %1
+...
+---
+name: test_fcanonicalize_v2bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fcanonicalize_v2bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST2]]
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL2]]
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[BITCAST4]](<2 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fcanonicalize_v2bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST2]]
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL2]]
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[BITCAST4]](<2 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fcanonicalize_v2bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST2]]
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX10-LABEL: name: test_fcanonicalize_v2bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST2]]
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD2]]
+ ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX10-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fcanonicalize_v2bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST]]
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST1]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST2]]
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fcanonicalize_v2bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST1]]
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST5]]
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST3]](bf16), [[BITCAST7]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ %0:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %1:_(<2 x bf16>) = G_FCANONICALIZE %0
+ S_NOP 0, implicit %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
index 490e60ee33a83..dd1beb3c638a5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
@@ -3,38 +3,53 @@
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GCN %s
---
-name: test_fconstant_s32
+name: test_fconstant_f32
body: |
bb.0:
- ; GCN-LABEL: name: test_fconstant_s32
- ; GCN: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
- ; GCN-NEXT: $vgpr0 = COPY [[C]](s32)
- %0:_(s32) = G_FCONSTANT float 1.0
+ ; GCN-LABEL: name: test_fconstant_f32
+ ; GCN: [[C:%[0-9]+]]:_(f32) = G_FCONSTANT float 1.000000e+00
+ ; GCN-NEXT: $vgpr0 = COPY [[C]](f32)
+ %0:_(f32) = G_FCONSTANT float 1.0
$vgpr0 = COPY %0
...
---
-name: test_fconstant_s64
+name: test_fconstant_f64
body: |
bb.0:
- ; GCN-LABEL: name: test_fconstant_s64
- ; GCN: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
- ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[C]](s64)
- %0:_(s64) = G_FCONSTANT double 1.0
+ ; GCN-LABEL: name: test_fconstant_f64
+ ; GCN: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.000000e+00
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[C]](f64)
+ %0:_(f64) = G_FCONSTANT double 1.0
$vgpr0_vgpr1 = COPY %0
...
---
-name: test_fconstant_s16
+name: test_fconstant_f16
body: |
bb.0:
- ; GCN-LABEL: name: test_fconstant_s16
- ; GCN: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 1.000000e+00
- ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
- ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
- %0:_(s16) = G_FCONSTANT half 1.0
- %1:_(s32) = G_ANYEXT %0
- $vgpr0 = COPY %1
+ ; GCN-LABEL: name: test_fconstant_f16
+ ; GCN: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half 1.000000e+00
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[C]](f16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ %0:_(f16) = G_FCONSTANT half 1.0
+ %1:_(i16) = G_BITCAST %0
+ %2:_(i32) = G_ANYEXT %1
+ $vgpr0 = COPY %2
+...
+
+---
+name: test_fconstant_bf16
+body: |
+ bb.0:
+ ; GCN-LABEL: name: test_fconstant_bf16
+ ; GCN: [[C:%[0-9]+]]:_(bf16) = G_FCONSTANT bfloat 1.000000e+00
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[C]](bf16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ %0:_(bf16) = G_FCONSTANT bfloat 1.0
+ %1:_(i16) = G_BITCAST %0
+ %2:_(i32) = G_ANYEXT %1
+ $vgpr0 = COPY %2
...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 3e555c10b0903..5ae605b47d3cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1,9 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
-# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GFX9 %s
+# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX10 %s
+# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX1250 %s
---
name: test_fma_f32
@@ -29,14 +30,14 @@ body: |
; VI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
; VI-NEXT: $vgpr0 = COPY [[FMA]](f32)
;
- ; GFX9-LABEL: name: test_fma_f32
- ; GFX9: liveins: $vgpr0, $vgpr1, $vgpr2
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(f32) = COPY $vgpr2
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FMA]](f32)
+ ; GCN-LABEL: name: test_fma_f32
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(f32) = COPY $vgpr2
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FMA]](f32)
%0:_(f32) = COPY $vgpr0
%1:_(f32) = COPY $vgpr1
%2:_(f32) = COPY $vgpr2
@@ -67,14 +68,14 @@ body: |
; VI-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
; VI-NEXT: $vgpr0_vgpr1 = COPY [[FMA]](f64)
;
- ; GFX9-LABEL: name: test_fma_f64
- ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(f64) = COPY $vgpr4_vgpr5
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FMA]](f64)
+ ; GCN-LABEL: name: test_fma_f64
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(f64) = COPY $vgpr4_vgpr5
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[FMA]](f64)
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f64) = COPY $vgpr2_vgpr3
%2:_(f64) = COPY $vgpr4_vgpr5
@@ -118,18 +119,18 @@ body: |
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMA]](f16)
; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
- ; GFX9-LABEL: name: test_fma_f16
- ; GFX9: liveins: $vgpr0, $vgpr1, $vgpr2
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY2]](i32)
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f16) = G_FMA [[TRUNC]], [[TRUNC1]], [[TRUNC2]]
- ; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMA]](f16)
- ; GFX9-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; GCN-LABEL: name: test_fma_f16
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GCN-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
+ ; GCN-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
+ ; GCN-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY2]](i32)
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(f16) = G_FMA [[TRUNC]], [[TRUNC1]], [[TRUNC2]]
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMA]](f16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
%0:_(i32) = COPY $vgpr0
%1:_(i32) = COPY $vgpr1
%2:_(i32) = COPY $vgpr1
@@ -189,6 +190,43 @@ body: |
; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v2f32
+ ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr4_vgpr5
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX10-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX10-NEXT: [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<2 x f32>)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV2]], [[UV4]]
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v2f32
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr4_vgpr5
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX11-NEXT: [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<2 x f32>)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV2]], [[UV4]]
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v2f32
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr4_vgpr5
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x f32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FMA]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = COPY $vgpr2_vgpr3
%2:_(<2 x f32>) = COPY $vgpr4_vgpr5
@@ -246,6 +284,59 @@ body: |
; GFX9-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v3f32
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX10-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX10-NEXT: [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32), [[UV8:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<3 x f32>)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV3]], [[UV6]]
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+ ; GFX10-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v3f32
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX11-NEXT: [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32), [[UV8:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<3 x f32>)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV3]], [[UV6]]
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+ ; GFX11-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v3f32
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV]](f32), [[UV1]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV2]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV3]](f32), [[UV4]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV5]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32), [[UV8:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<3 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV6]](f32), [[UV7]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV8]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x f32>) = G_FMA [[BUILD_VECTOR]], [[BUILD_VECTOR2]], [[BUILD_VECTOR4]]
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f32>) = G_FMA [[BUILD_VECTOR1]], [[BUILD_VECTOR3]], [[BUILD_VECTOR5]]
+ ; GFX1250-NEXT: [[UV9:%[0-9]+]]:_(f32), [[UV10:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FMA]](<2 x f32>)
+ ; GFX1250-NEXT: [[UV11:%[0-9]+]]:_(f32), [[UV12:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FMA1]](<2 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[UV9]](f32), [[UV10]](f32), [[UV11]](f32)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR6]](<3 x f32>)
%0:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<3 x f32>) = COPY $vgpr6_vgpr7_vgpr8
@@ -306,6 +397,52 @@ body: |
; GFX9-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32), [[FMA3]](f32)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x f32>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v4f32
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<4 x f32>)
+ ; GFX10-NEXT: [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32), [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<4 x f32>)
+ ; GFX10-NEXT: [[UV8:%[0-9]+]]:_(f32), [[UV9:%[0-9]+]]:_(f32), [[UV10:%[0-9]+]]:_(f32), [[UV11:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<4 x f32>)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV4]], [[UV8]]
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+ ; GFX10-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+ ; GFX10-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32), [[FMA3]](f32)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v4f32
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<4 x f32>)
+ ; GFX11-NEXT: [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32), [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<4 x f32>)
+ ; GFX11-NEXT: [[UV8:%[0-9]+]]:_(f32), [[UV9:%[0-9]+]]:_(f32), [[UV10:%[0-9]+]]:_(f32), [[UV11:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY2]](<4 x f32>)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[UV]], [[UV4]], [[UV8]]
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+ ; GFX11-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+ ; GFX11-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[FMA]](f32), [[FMA1]](f32), [[FMA2]](f32), [[FMA3]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v4f32
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(<4 x f32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(<2 x f32>), [[UV1:%[0-9]+]]:_(<2 x f32>) = G_UNMERGE_VALUES [[COPY]](<4 x f32>)
+ ; GFX1250-NEXT: [[UV2:%[0-9]+]]:_(<2 x f32>), [[UV3:%[0-9]+]]:_(<2 x f32>) = G_UNMERGE_VALUES [[COPY1]](<4 x f32>)
+ ; GFX1250-NEXT: [[UV4:%[0-9]+]]:_(<2 x f32>), [[UV5:%[0-9]+]]:_(<2 x f32>) = G_UNMERGE_VALUES [[COPY2]](<4 x f32>)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x f32>) = G_FMA [[UV]], [[UV2]], [[UV4]]
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f32>) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f32>) = G_CONCAT_VECTORS [[FMA]](<2 x f32>), [[FMA1]](<2 x f32>)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x f32>)
%0:_(<4 x f32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<4 x f32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
%2:_(<4 x f32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
@@ -347,19 +484,19 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMA]](f64), [[FMA1]](f64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
;
- ; GFX9-LABEL: name: test_fma_v2f64
- ; GFX9: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
- ; GFX9-NEXT: [[UV4:%[0-9]+]]:_(f64), [[UV5:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY2]](<2 x f64>)
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[UV]], [[UV2]], [[UV4]]
- ; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(f64) = G_FMA [[UV1]], [[UV3]], [[UV5]]
- ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMA]](f64), [[FMA1]](f64)
- ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
+ ; GCN-LABEL: name: test_fma_v2f64
+ ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
+ ; GCN-NEXT: [[UV4:%[0-9]+]]:_(f64), [[UV5:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY2]](<2 x f64>)
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[UV]], [[UV2]], [[UV4]]
+ ; GCN-NEXT: [[FMA1:%[0-9]+]]:_(f64) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+ ; GCN-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMA]](f64), [[FMA1]](f64)
+ ; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
%2:_(<2 x f64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
@@ -437,14 +574,14 @@ body: |
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x f16>)
;
- ; GFX9-LABEL: name: test_fma_v2f16
- ; GFX9: liveins: $vgpr0, $vgpr1, $vgpr2
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
- ; GFX9-NEXT: $vgpr0 = COPY [[FMA]](<2 x f16>)
+ ; GCN-LABEL: name: test_fma_v2f16
+ ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FMA]](<2 x f16>)
%0:_(<2 x f16>) = COPY $vgpr0
%1:_(<2 x f16>) = COPY $vgpr1
%2:_(<2 x f16>) = COPY $vgpr2
@@ -586,6 +723,78 @@ body: |
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v3f16
+ ; GFX10: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX10-NEXT: {{ $}}
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX10-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX10-NEXT: [[UV6:%[0-9]+]]:_(<2 x f16>), [[UV7:%[0-9]+]]:_(<2 x f16>), [[UV8:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY2]](<6 x f16>)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV]], [[UV3]], [[UV6]]
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](<2 x f16>)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](<2 x f16>)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v3f16
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX11-NEXT: [[UV6:%[0-9]+]]:_(<2 x f16>), [[UV7:%[0-9]+]]:_(<2 x f16>), [[UV8:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY2]](<6 x f16>)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV]], [[UV3]], [[UV6]]
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v3f16
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr6_vgpr7_vgpr8
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16), [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(<2 x f16>), [[UV7:%[0-9]+]]:_(<2 x f16>), [[UV8:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV2]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[UV9:%[0-9]+]]:_(f16), [[UV10:%[0-9]+]]:_(f16), [[UV11:%[0-9]+]]:_(f16), [[UV12:%[0-9]+]]:_(f16), [[UV13:%[0-9]+]]:_(f16), [[UV14:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[UV15:%[0-9]+]]:_(<2 x f16>), [[UV16:%[0-9]+]]:_(<2 x f16>), [[UV17:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV11]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[UV18:%[0-9]+]]:_(f16), [[UV19:%[0-9]+]]:_(f16), [[UV20:%[0-9]+]]:_(f16), [[UV21:%[0-9]+]]:_(f16), [[UV22:%[0-9]+]]:_(f16), [[UV23:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY2]](<6 x f16>)
+ ; GFX1250-NEXT: [[UV24:%[0-9]+]]:_(<2 x f16>), [[UV25:%[0-9]+]]:_(<2 x f16>), [[UV26:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY2]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV20]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV6]], [[UV15]], [[UV24]]
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f16>) = G_FMA [[BUILD_VECTOR]], [[BUILD_VECTOR1]], [[BUILD_VECTOR2]]
+ ; GFX1250-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[FMA]](<2 x f16>), [[FMA1]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
%0:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<6 x f16>) = COPY $vgpr6_vgpr7_vgpr8
@@ -723,22 +932,1003 @@ body: |
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
;
- ; GFX9-LABEL: name: test_fma_v4f16
- ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
- ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr4_vgpr5
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
- ; GFX9-NEXT: [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY2]](<4 x f16>)
- ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV]], [[UV2]], [[UV4]]
- ; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV1]], [[UV3]], [[UV5]]
- ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FMA]](<2 x f16>), [[FMA1]](<2 x f16>)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
+ ; GCN-LABEL: name: test_fma_v4f16
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr4_vgpr5
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
+ ; GCN-NEXT: [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY2]](<4 x f16>)
+ ; GCN-NEXT: [[FMA:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV]], [[UV2]], [[UV4]]
+ ; GCN-NEXT: [[FMA1:%[0-9]+]]:_(<2 x f16>) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+ ; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FMA]](<2 x f16>), [[FMA1]](<2 x f16>)
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
%0:_(<4 x f16>) = COPY $vgpr0_vgpr1
%1:_(<4 x f16>) = COPY $vgpr2_vgpr3
%2:_(<4 x f16>) = COPY $vgpr4_vgpr5
%3:_(<4 x f16>) = G_FMA %0, %1, %2
$vgpr0_vgpr1 = COPY %3
...
+
+---
+name: test_fma_bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fma_bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; VI-LABEL: name: test_fma_bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX9-LABEL: name: test_fma_bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX10-LABEL: name: test_fma_bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX11-LABEL: name: test_fma_bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX1250-LABEL: name: test_fma_bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF2:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF2]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST7]](bf16)
+ %0:_(bf16) = G_IMPLICIT_DEF
+ %1:_(bf16) = G_IMPLICIT_DEF
+ %2:_(bf16) = G_IMPLICIT_DEF
+ %3:_(bf16) = G_FMA %0, %1, %2
+ S_NOP 0, implicit %3
+...
+
+---
+name: test_fma_v2bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fma_v2bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; SI-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL6]]
+ ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[BITCAST8]](<2 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fma_v2bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; VI-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL6]]
+ ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[BITCAST8]](<2 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fma_v2bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v2bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX10-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v2bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX11-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v2bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST10]](i16)
+ ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
+ ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST9]], [[BITCAST11]], [[BITCAST13]]
+ ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST14]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST14]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST14]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST7]](bf16), [[BITCAST15]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ %0:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %1:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %2:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %3:_(<2 x bf16>) = G_FMA %0, %1, %2
+ S_NOP 0, implicit %3
+...
+
+---
+name: test_fma_v4bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fma_v4bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; SI-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; SI-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST8]], [[BITCAST9]], [[BITCAST10]]
+ ; SI-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; SI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD4]]
+ ; SI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST12:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; SI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; SI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; SI-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST12]], [[BITCAST13]], [[BITCAST14]]
+ ; SI-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; SI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; SI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; SI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; SI-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL12]]
+ ; SI-NEXT: [[BITCAST16:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; SI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL13]]
+ ; SI-NEXT: [[BITCAST17:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST16]](<2 x bf16>), [[BITCAST17]](<2 x bf16>)
+ ; SI-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fma_v4bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; VI-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; VI-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST8]], [[BITCAST9]], [[BITCAST10]]
+ ; VI-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; VI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; VI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD4]]
+ ; VI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST12:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; VI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; VI-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST12]], [[BITCAST13]], [[BITCAST14]]
+ ; VI-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; VI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; VI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; VI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; VI-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL12]]
+ ; VI-NEXT: [[BITCAST16:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; VI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL13]]
+ ; VI-NEXT: [[BITCAST17:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST16]](<2 x bf16>), [[BITCAST17]](<2 x bf16>)
+ ; VI-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fma_v4bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX9-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX9-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; GFX9-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST8]], [[BITCAST9]], [[BITCAST10]]
+ ; GFX9-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; GFX9-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX9-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX9-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD4]]
+ ; GFX9-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX9-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX9-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST12:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; GFX9-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; GFX9-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; GFX9-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST12]], [[BITCAST13]], [[BITCAST14]]
+ ; GFX9-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX9-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX9-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX9-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX9-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX10-LABEL: name: test_fma_v4bf16
+ ; GFX10: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX10-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX10-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX10-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX10-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX10-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX10-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX10-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX10-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX10-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX10-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX10-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX10-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX10-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX10-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; GFX10-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST8]], [[BITCAST9]], [[BITCAST10]]
+ ; GFX10-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; GFX10-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; GFX10-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; GFX10-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX10-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX10-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD4]]
+ ; GFX10-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX10-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX10-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST12:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; GFX10-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; GFX10-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX10-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; GFX10-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST12]], [[BITCAST13]], [[BITCAST14]]
+ ; GFX10-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; GFX10-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX10-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX10-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX10-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX10-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX10-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; GFX10-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX10-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX10-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fma_v4bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST]], [[BITCAST1]], [[BITCAST2]]
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST3]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX11-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX11-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST4]], [[BITCAST5]], [[BITCAST6]]
+ ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST7]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST7]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX11-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX11-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; GFX11-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST8]], [[BITCAST9]], [[BITCAST10]]
+ ; GFX11-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; GFX11-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; GFX11-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; GFX11-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX11-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX11-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD4]]
+ ; GFX11-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX11-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX11-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST12:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; GFX11-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; GFX11-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST14:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; GFX11-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST12]], [[BITCAST13]], [[BITCAST14]]
+ ; GFX11-NEXT: [[BITCAST15:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; GFX11-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST15]], [[C1]]
+ ; GFX11-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST15]], [[C]](i32)
+ ; GFX11-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX11-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX11-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST15]], [[ADD6]]
+ ; GFX11-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX11-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX11-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fma_v4bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST10]](i16)
+ ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
+ ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST9]], [[BITCAST11]], [[BITCAST13]]
+ ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST14]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST14]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST14]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST16]](i16)
+ ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
+ ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT8:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
+ ; GFX1250-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT8]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
+ ; GFX1250-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST17]], [[BITCAST19]], [[BITCAST21]]
+ ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
+ ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD4]]
+ ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX1250-NEXT: [[BITCAST24:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT9:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST24]](i16)
+ ; GFX1250-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT9]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST25:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
+ ; GFX1250-NEXT: [[BITCAST26:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT10:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST26]](i16)
+ ; GFX1250-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT10]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST27:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
+ ; GFX1250-NEXT: [[BITCAST28:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT11:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST28]](i16)
+ ; GFX1250-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT11]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST29:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
+ ; GFX1250-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST25]], [[BITCAST27]], [[BITCAST29]]
+ ; GFX1250-NEXT: [[BITCAST30:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
+ ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST30]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST30]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST30]], [[ADD6]]
+ ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX1250-NEXT: [[BITCAST31:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST7]](bf16), [[BITCAST15]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST23]](bf16), [[BITCAST31]](bf16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ %0:_(<4 x bf16>) = G_IMPLICIT_DEF
+ %1:_(<4 x bf16>) = G_IMPLICIT_DEF
+ %2:_(<4 x bf16>) = G_IMPLICIT_DEF
+ %3:_(<4 x bf16>) = G_FMA %0, %1, %2
+ S_NOP 0, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 330c95d020411..57d941daf7718 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -1,9 +1,9 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
-# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9PLUS %s
-# RUN: llc -mtriple=amdgpu10.10-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9PLUS %s
-# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GFX9PLUS %s
+# RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: llc -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck -check-prefixes=GCN,GFX1250 %s
---
name: test_fmul_f32
@@ -27,13 +27,13 @@ body: |
; VI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[COPY]], [[COPY1]]
; VI-NEXT: $vgpr0 = COPY [[FMUL]](f32)
;
- ; GFX9PLUS-LABEL: name: test_fmul_f32
- ; GFX9PLUS: liveins: $vgpr0, $vgpr1
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[COPY]], [[COPY1]]
- ; GFX9PLUS-NEXT: $vgpr0 = COPY [[FMUL]](f32)
+ ; GCN-LABEL: name: test_fmul_f32
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f32) = COPY $vgpr1
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FMUL]](f32)
%0:_(f32) = COPY $vgpr0
%1:_(f32) = COPY $vgpr1
%2:_(f32) = G_FMUL %0, %1
@@ -61,13 +61,13 @@ body: |
; VI-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[COPY]], [[COPY1]]
; VI-NEXT: $vgpr0_vgpr1 = COPY [[FMUL]](f64)
;
- ; GFX9PLUS-LABEL: name: test_fmul_f64
- ; GFX9PLUS: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[COPY]], [[COPY1]]
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[FMUL]](f64)
+ ; GCN-LABEL: name: test_fmul_f64
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(f64) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[FMUL]](f64)
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f64) = COPY $vgpr2_vgpr3
%2:_(f64) = G_FMUL %0, %1
@@ -105,16 +105,16 @@ body: |
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMUL]](f16)
; VI-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
;
- ; GFX9PLUS-LABEL: name: test_fmul_f16
- ; GFX9PLUS: liveins: $vgpr0, $vgpr1
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
- ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f16) = G_FMUL [[TRUNC]], [[TRUNC1]]
- ; GFX9PLUS-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMUL]](f16)
- ; GFX9PLUS-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; GCN-LABEL: name: test_fmul_f16
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GCN-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
+ ; GCN-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(f16) = G_FMUL [[TRUNC]], [[TRUNC1]]
+ ; GCN-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FMUL]](f16)
+ ; GCN-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
%0:_(i32) = COPY $vgpr0
%1:_(i32) = COPY $vgpr1
%2:_(f16) = G_TRUNC %0
@@ -155,17 +155,37 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v2f32
- ; GFX9PLUS: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
- ; GFX9PLUS-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV2]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV3]]
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ; GFX9-LABEL: name: test_fmul_v2f32
+ ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV2]]
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV3]]
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v2f32
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV2]]
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV3]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v2f32
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f32>) = G_FMUL [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FMUL]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = COPY $vgpr2_vgpr3
%2:_(<2 x f32>) = G_FMUL %0, %1
@@ -202,17 +222,37 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v2f32_flags
- ; GFX9PLUS: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
- ; GFX9PLUS-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV]], [[UV2]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV1]], [[UV3]]
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ; GFX9-LABEL: name: test_fmul_v2f32_flags
+ ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV]], [[UV2]]
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV1]], [[UV3]]
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v2f32_flags
+ ; GFX11: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
+ ; GFX11-NEXT: [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<2 x f32>)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV]], [[UV2]]
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = nnan G_FMUL [[UV1]], [[UV3]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v2f32_flags
+ ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr2_vgpr3
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f32>) = nnan G_FMUL [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[FMUL]](<2 x f32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f32>) = COPY $vgpr2_vgpr3
%2:_(<2 x f32>) = nnan G_FMUL %0, %1
@@ -251,18 +291,50 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32), [[FMUL2]](f32)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v3f32
- ; GFX9PLUS: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
- ; GFX9PLUS-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV3]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV4]]
- ; GFX9PLUS-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[UV2]], [[UV5]]
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32), [[FMUL2]](f32)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ; GFX9-LABEL: name: test_fmul_v3f32
+ ; GFX9: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX9-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV3]]
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV4]]
+ ; GFX9-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[UV2]], [[UV5]]
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32), [[FMUL2]](f32)
+ ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v3f32
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[UV]], [[UV3]]
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[UV1]], [[UV4]]
+ ; GFX11-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[UV2]], [[UV5]]
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[FMUL]](f32), [[FMUL1]](f32), [[FMUL2]](f32)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x f32>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v3f32
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<3 x f32>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV]](f32), [[UV1]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV2]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[UV3:%[0-9]+]]:_(f32), [[UV4:%[0-9]+]]:_(f32), [[UV5:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY1]](<3 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV3]](f32), [[UV4]](f32)
+ ; GFX1250-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[UV5]](f32), [[DEF]](f32)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f32>) = G_FMUL [[BUILD_VECTOR]], [[BUILD_VECTOR2]]
+ ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f32>) = G_FMUL [[BUILD_VECTOR1]], [[BUILD_VECTOR3]]
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(f32), [[UV7:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FMUL]](<2 x f32>)
+ ; GFX1250-NEXT: [[UV8:%[0-9]+]]:_(f32), [[UV9:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[FMUL1]](<2 x f32>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x f32>) = G_BUILD_VECTOR [[UV6]](f32), [[UV7]](f32), [[UV8]](f32)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR4]](<3 x f32>)
%0:_(<3 x f32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<3 x f32>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<3 x f32>) = G_FMUL %0, %1
@@ -299,17 +371,17 @@ body: |
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMUL]](f64), [[FMUL1]](f64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v2f64
- ; GFX9PLUS: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
- ; GFX9PLUS-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[UV]], [[UV2]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(f64) = G_FMUL [[UV1]], [[UV3]]
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMUL]](f64), [[FMUL1]](f64)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
+ ; GCN-LABEL: name: test_fmul_v2f64
+ ; GCN: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(f64), [[UV3:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY1]](<2 x f64>)
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[UV]], [[UV2]]
+ ; GCN-NEXT: [[FMUL1:%[0-9]+]]:_(f64) = G_FMUL [[UV1]], [[UV3]]
+ ; GCN-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f64>) = G_BUILD_VECTOR [[FMUL]](f64), [[FMUL1]](f64)
+ ; GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x f64>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
%2:_(<2 x f64>) = G_FMUL %0, %1
@@ -374,13 +446,13 @@ body: |
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x f16>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v2f16
- ; GFX9PLUS: liveins: $vgpr0, $vgpr1
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[COPY]], [[COPY1]]
- ; GFX9PLUS-NEXT: $vgpr0 = COPY [[FMUL]](<2 x f16>)
+ ; GCN-LABEL: name: test_fmul_v2f16
+ ; GCN: liveins: $vgpr0, $vgpr1
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[COPY]], [[COPY1]]
+ ; GCN-NEXT: $vgpr0 = COPY [[FMUL]](<2 x f16>)
%0:_(<2 x f16>) = COPY $vgpr0
%1:_(<2 x f16>) = COPY $vgpr1
%2:_(<2 x f16>) = G_FMUL %0, %1
@@ -478,28 +550,69 @@ body: |
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v3f16
- ; GFX9PLUS: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
- ; GFX9PLUS-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX9PLUS-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV]], [[UV3]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV1]], [[UV4]]
- ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](<2 x f16>)
- ; GFX9PLUS-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
- ; GFX9PLUS-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9PLUS-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
- ; GFX9PLUS-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](<2 x f16>)
- ; GFX9PLUS-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
- ; GFX9PLUS-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
- ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ; GFX9-LABEL: name: test_fmul_v3f16
+ ; GFX9: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX9-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV]], [[UV3]]
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV1]], [[UV4]]
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](<2 x f16>)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v3f16
+ ; GFX11: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX11-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>), [[UV2:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[UV3:%[0-9]+]]:_(<2 x f16>), [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV]], [[UV3]]
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV1]], [[UV4]]
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST]](i32)
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](<2 x f16>)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST1]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x f16>), [[BUILD_VECTOR1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v3f16
+ ; GFX1250: liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16), [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[UV6:%[0-9]+]]:_(<2 x f16>), [[UV7:%[0-9]+]]:_(<2 x f16>), [[UV8:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV2]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[UV9:%[0-9]+]]:_(f16), [[UV10:%[0-9]+]]:_(f16), [[UV11:%[0-9]+]]:_(f16), [[UV12:%[0-9]+]]:_(f16), [[UV13:%[0-9]+]]:_(f16), [[UV14:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[UV15:%[0-9]+]]:_(<2 x f16>), [[UV16:%[0-9]+]]:_(<2 x f16>), [[UV17:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<6 x f16>)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[UV11]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV6]], [[UV15]]
+ ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[DEF]](f16), [[DEF]](f16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[FMUL]](<2 x f16>), [[FMUL1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>)
+ ; GFX1250-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x f16>)
%0:_(<6 x f16>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<6 x f16>) = COPY $vgpr3_vgpr4_vgpr5
%2:_(<3 x f16>), %3:_(<3 x f16>) = G_UNMERGE_VALUES %0
@@ -611,19 +724,762 @@ body: |
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
;
- ; GFX9PLUS-LABEL: name: test_fmul_v4f16
- ; GFX9PLUS: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: {{ $}}
- ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
- ; GFX9PLUS-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
- ; GFX9PLUS-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
- ; GFX9PLUS-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
- ; GFX9PLUS-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV]], [[UV2]]
- ; GFX9PLUS-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV1]], [[UV3]]
- ; GFX9PLUS-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FMUL]](<2 x f16>), [[FMUL1]](<2 x f16>)
- ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
+ ; GCN-LABEL: name: test_fmul_v4f16
+ ; GCN: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr0_vgpr1
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:_(<4 x f16>) = COPY $vgpr2_vgpr3
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY]](<4 x f16>)
+ ; GCN-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[COPY1]](<4 x f16>)
+ ; GCN-NEXT: [[FMUL:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV]], [[UV2]]
+ ; GCN-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x f16>) = G_FMUL [[UV1]], [[UV3]]
+ ; GCN-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[FMUL]](<2 x f16>), [[FMUL1]](<2 x f16>)
+ ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x f16>)
%0:_(<4 x f16>) = COPY $vgpr0_vgpr1
%1:_(<4 x f16>) = COPY $vgpr2_vgpr3
%2:_(<4 x f16>) = G_FMUL %0, %1
$vgpr0_vgpr1 = COPY %2
...
+
+---
+name: test_fmul_bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fmul_bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; VI-LABEL: name: test_fmul_bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX9-LABEL: name: test_fmul_bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX11-LABEL: name: test_fmul_bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: S_NOP 0, implicit [[TRUNC]](bf16)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ %0:_(bf16) = G_IMPLICIT_DEF
+ %1:_(bf16) = G_IMPLICIT_DEF
+ %2:_(bf16) = G_FMUL %0, %1
+ S_NOP 0, implicit %2
+...
+
+---
+name: test_fmul_v2bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fmul_v2bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL4]]
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; SI-NEXT: S_NOP 0, implicit [[BITCAST6]](<2 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fmul_v2bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL4]]
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR2]](i32)
+ ; VI-NEXT: S_NOP 0, implicit [[BITCAST6]](<2 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fmul_v2bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v2bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v2bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST7]], [[BITCAST9]]
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ %0:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %1:_(<2 x bf16>) = G_IMPLICIT_DEF
+ %2:_(<2 x bf16>) = G_FMUL %0, %1
+ S_NOP 0, implicit %2
+...
+
+---
+name: test_fmul_v4bf16
+body: |
+ bb.0:
+ ; SI-LABEL: name: test_fmul_v4bf16
+ ; SI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; SI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; SI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; SI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; SI-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; SI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; SI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; SI-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST6]], [[BITCAST7]]
+ ; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST8]], [[C1]]
+ ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; SI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST8]], [[ADD4]]
+ ; SI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; SI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; SI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; SI-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST9]], [[BITCAST10]]
+ ; SI-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
+ ; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; SI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; SI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; SI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD6]]
+ ; SI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
+ ; SI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; SI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL8]]
+ ; SI-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL9]]
+ ; SI-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST12]](<2 x bf16>), [[BITCAST13]](<2 x bf16>)
+ ; SI-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; VI-LABEL: name: test_fmul_v4bf16
+ ; VI: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; VI-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; VI-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; VI-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; VI-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; VI-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; VI-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; VI-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; VI-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST6]], [[BITCAST7]]
+ ; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST8]], [[C1]]
+ ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; VI-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; VI-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST8]], [[ADD4]]
+ ; VI-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; VI-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; VI-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; VI-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST9]], [[BITCAST10]]
+ ; VI-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
+ ; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; VI-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; VI-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; VI-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD6]]
+ ; VI-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
+ ; VI-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; VI-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
+ ; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL8]]
+ ; VI-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR4]](i32)
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL9]]
+ ; VI-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x bf16>) = G_BITCAST [[OR5]](i32)
+ ; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BITCAST12]](<2 x bf16>), [[BITCAST13]](<2 x bf16>)
+ ; VI-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX9-LABEL: name: test_fmul_v4bf16
+ ; GFX9: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX9-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX9-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX9-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX9-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX9-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; GFX9-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX9-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX9-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; GFX9-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX9-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST6]], [[BITCAST7]]
+ ; GFX9-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
+ ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST8]], [[C1]]
+ ; GFX9-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX9-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX9-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST8]], [[ADD4]]
+ ; GFX9-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX9-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX9-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX9-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX9-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX9-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST9]], [[BITCAST10]]
+ ; GFX9-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
+ ; GFX9-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX9-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX9-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD6]]
+ ; GFX9-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
+ ; GFX9-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX9-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX11-LABEL: name: test_fmul_v4bf16
+ ; GFX11: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX11-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX11-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX11-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST]], [[BITCAST1]]
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX11-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX11-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
+ ; GFX11-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX11-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX11-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX11-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX11-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX11-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
+ ; GFX11-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX11-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX11-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX11-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX11-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX11-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST3]], [[BITCAST4]]
+ ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX11-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST5]], [[C1]]
+ ; GFX11-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX11-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX11-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX11-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST5]], [[ADD2]]
+ ; GFX11-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX11-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX11-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX11-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX11-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST6]], [[BITCAST7]]
+ ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
+ ; GFX11-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST8]], [[C1]]
+ ; GFX11-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; GFX11-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX11-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX11-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST8]], [[ADD4]]
+ ; GFX11-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX11-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX11-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX11-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; GFX11-NEXT: [[BITCAST10:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX11-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST9]], [[BITCAST10]]
+ ; GFX11-NEXT: [[BITCAST11:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
+ ; GFX11-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST11]], [[C1]]
+ ; GFX11-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST11]], [[C]](i32)
+ ; GFX11-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX11-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX11-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST11]], [[ADD6]]
+ ; GFX11-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
+ ; GFX11-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX11-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(bf16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC]](bf16), [[TRUNC1]](bf16)
+ ; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[TRUNC2]](bf16), [[TRUNC3]](bf16)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX11-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ ;
+ ; GFX1250-LABEL: name: test_fmul_v4bf16
+ ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
+ ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
+ ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
+ ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
+ ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
+ ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
+ ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
+ ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
+ ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
+ ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
+ ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
+ ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
+ ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
+ ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST7]], [[BITCAST9]]
+ ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
+ ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
+ ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
+ ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
+ ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
+ ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
+ ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST14]](i16)
+ ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
+ ; GFX1250-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST13]], [[BITCAST15]]
+ ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
+ ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST16]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST16]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
+ ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST16]], [[ADD4]]
+ ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
+ ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
+ ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
+ ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
+ ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
+ ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
+ ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
+ ; GFX1250-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST19]], [[BITCAST21]]
+ ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
+ ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
+ ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
+ ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
+ ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD6]]
+ ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
+ ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
+ ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
+ ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
+ ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST17]](bf16), [[BITCAST23]](bf16)
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
+ %0:_(<4 x bf16>) = G_IMPLICIT_DEF
+ %1:_(<4 x bf16>) = G_IMPLICIT_DEF
+ %2:_(<4 x bf16>) = G_FMUL %0, %1
+ S_NOP 0, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
index 2bd202f4c71fb..0adda870b0819 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx7.mir
@@ -41,35 +41,35 @@ body: |
; GFX7-LABEL: name: and_v2i16
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR]](<2 x s32>)
- ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BUILD_VECTOR1]](<2 x s32>)
- ; GFX7-NEXT: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[TRUNC]], [[TRUNC1]]
- ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[AND]](<2 x s16>)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR]](<2 x i32>)
+ ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX7-NEXT: [[AND:%[0-9]+]]:_(<2 x i16>) = G_AND [[TRUNC]], [[TRUNC1]]
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[AND]](<2 x i16>)
; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST]](i32)
; GFX7-NEXT: $vgpr1 = COPY [[LSHR]](i32)
; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
- %3:_(s32) = COPY $vgpr0
- %4:_(s32) = COPY $vgpr1
- %5:_(<2 x s32>) = G_BUILD_VECTOR %3(s32), %4(s32)
- %0:_(<2 x s16>) = G_TRUNC %5(<2 x s32>)
- %6:_(s32) = COPY $vgpr2
- %7:_(s32) = COPY $vgpr3
- %8:_(<2 x s32>) = G_BUILD_VECTOR %6(s32), %7(s32)
- %1:_(<2 x s16>) = G_TRUNC %8(<2 x s32>)
- %9:_(<2 x s16>) = G_AND %0, %1
- %13:_(s16), %14:_(s16) = G_UNMERGE_VALUES %9(<2 x s16>)
- %11:_(s32) = G_ANYEXT %13(s16)
- %12:_(s32) = G_ANYEXT %14(s16)
- $vgpr0 = COPY %11(s32)
- $vgpr1 = COPY %12(s32)
+ %3:_(i32) = COPY $vgpr0
+ %4:_(i32) = COPY $vgpr1
+ %5:_(<2 x i32>) = G_BUILD_VECTOR %3(i32), %4(i32)
+ %0:_(<2 x i16>) = G_TRUNC %5(<2 x i32>)
+ %6:_(i32) = COPY $vgpr2
+ %7:_(i32) = COPY $vgpr3
+ %8:_(<2 x i32>) = G_BUILD_VECTOR %6(i32), %7(i32)
+ %1:_(<2 x i16>) = G_TRUNC %8(<2 x i32>)
+ %9:_(<2 x i16>) = G_AND %0, %1
+ %13:_(i16), %14:_(i16) = G_UNMERGE_VALUES %9(<2 x i16>)
+ %11:_(i32) = G_ANYEXT %13(i16)
+ %12:_(i32) = G_ANYEXT %14(i16)
+ $vgpr0 = COPY %11(i32)
+ $vgpr1 = COPY %12(i32)
SI_RETURN implicit $vgpr0, implicit $vgpr1
...
@@ -83,37 +83,37 @@ body: |
; GFX7-LABEL: name: add_v3i16
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
- ; GFX7-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY3]]
- ; GFX7-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY4]]
- ; GFX7-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY5]]
- ; GFX7-NEXT: $vgpr0 = COPY [[ADD]](s32)
- ; GFX7-NEXT: $vgpr1 = COPY [[ADD1]](s32)
- ; GFX7-NEXT: $vgpr2 = COPY [[ADD2]](s32)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
+ ; GFX7-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[COPY]], [[COPY3]]
+ ; GFX7-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[COPY1]], [[COPY4]]
+ ; GFX7-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[COPY2]], [[COPY5]]
+ ; GFX7-NEXT: $vgpr0 = COPY [[ADD]](i32)
+ ; GFX7-NEXT: $vgpr1 = COPY [[ADD1]](i32)
+ ; GFX7-NEXT: $vgpr2 = COPY [[ADD2]](i32)
; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
- %3:_(s32) = COPY $vgpr0
- %4:_(s32) = COPY $vgpr1
- %5:_(s32) = COPY $vgpr2
- %6:_(<3 x s32>) = G_BUILD_VECTOR %3(s32), %4(s32), %5(s32)
- %0:_(<3 x s16>) = G_TRUNC %6(<3 x s32>)
- %7:_(s32) = COPY $vgpr3
- %8:_(s32) = COPY $vgpr4
- %9:_(s32) = COPY $vgpr5
- %10:_(<3 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32), %9(s32)
- %1:_(<3 x s16>) = G_TRUNC %10(<3 x s32>)
- %11:_(<3 x s16>) = G_ADD %0, %1
- %16:_(s16), %17:_(s16), %18:_(s16) = G_UNMERGE_VALUES %11(<3 x s16>)
- %13:_(s32) = G_ANYEXT %16(s16)
- %14:_(s32) = G_ANYEXT %17(s16)
- %15:_(s32) = G_ANYEXT %18(s16)
- $vgpr0 = COPY %13(s32)
- $vgpr1 = COPY %14(s32)
- $vgpr2 = COPY %15(s32)
+ %3:_(i32) = COPY $vgpr0
+ %4:_(i32) = COPY $vgpr1
+ %5:_(i32) = COPY $vgpr2
+ %6:_(<3 x i32>) = G_BUILD_VECTOR %3(i32), %4(i32), %5(i32)
+ %0:_(<3 x i16>) = G_TRUNC %6(<3 x i32>)
+ %7:_(i32) = COPY $vgpr3
+ %8:_(i32) = COPY $vgpr4
+ %9:_(i32) = COPY $vgpr5
+ %10:_(<3 x i32>) = G_BUILD_VECTOR %7(i32), %8(i32), %9(i32)
+ %1:_(<3 x i16>) = G_TRUNC %10(<3 x i32>)
+ %11:_(<3 x i16>) = G_ADD %0, %1
+ %16:_(i16), %17:_(i16), %18:_(i16) = G_UNMERGE_VALUES %11(<3 x i16>)
+ %13:_(i32) = G_ANYEXT %16(i16)
+ %14:_(i32) = G_ANYEXT %17(i16)
+ %15:_(i32) = G_ANYEXT %18(i16)
+ $vgpr0 = COPY %13(i32)
+ $vgpr1 = COPY %14(i32)
+ $vgpr2 = COPY %15(i32)
SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
...
@@ -127,41 +127,41 @@ body: |
; GFX7-LABEL: name: shl_v3i16
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
- ; GFX7-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX7-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
- ; GFX7-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[AND1]](s32)
- ; GFX7-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
- ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[AND2]](s32)
- ; GFX7-NEXT: $vgpr0 = COPY [[SHL]](s32)
- ; GFX7-NEXT: $vgpr1 = COPY [[SHL1]](s32)
- ; GFX7-NEXT: $vgpr2 = COPY [[SHL2]](s32)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX7-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY]], [[AND]](i32)
+ ; GFX7-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[COPY1]], [[AND1]](i32)
+ ; GFX7-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY5]], [[C]]
+ ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[COPY2]], [[AND2]](i32)
+ ; GFX7-NEXT: $vgpr0 = COPY [[SHL]](i32)
+ ; GFX7-NEXT: $vgpr1 = COPY [[SHL1]](i32)
+ ; GFX7-NEXT: $vgpr2 = COPY [[SHL2]](i32)
; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
- %3:_(s32) = COPY $vgpr0
- %4:_(s32) = COPY $vgpr1
- %5:_(s32) = COPY $vgpr2
- %6:_(<3 x s32>) = G_BUILD_VECTOR %3(s32), %4(s32), %5(s32)
- %0:_(<3 x s16>) = G_TRUNC %6(<3 x s32>)
- %7:_(s32) = COPY $vgpr3
- %8:_(s32) = COPY $vgpr4
- %9:_(s32) = COPY $vgpr5
- %10:_(<3 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32), %9(s32)
- %1:_(<3 x s16>) = G_TRUNC %10(<3 x s32>)
- %11:_(<3 x s16>) = G_SHL %0, %1(<3 x s16>)
- %16:_(s16), %17:_(s16), %18:_(s16) = G_UNMERGE_VALUES %11(<3 x s16>)
- %13:_(s32) = G_ANYEXT %16(s16)
- %14:_(s32) = G_ANYEXT %17(s16)
- %15:_(s32) = G_ANYEXT %18(s16)
- $vgpr0 = COPY %13(s32)
- $vgpr1 = COPY %14(s32)
- $vgpr2 = COPY %15(s32)
+ %3:_(i32) = COPY $vgpr0
+ %4:_(i32) = COPY $vgpr1
+ %5:_(i32) = COPY $vgpr2
+ %6:_(<3 x i32>) = G_BUILD_VECTOR %3(i32), %4(i32), %5(i32)
+ %0:_(<3 x i16>) = G_TRUNC %6(<3 x i32>)
+ %7:_(i32) = COPY $vgpr3
+ %8:_(i32) = COPY $vgpr4
+ %9:_(i32) = COPY $vgpr5
+ %10:_(<3 x i32>) = G_BUILD_VECTOR %7(i32), %8(i32), %9(i32)
+ %1:_(<3 x i16>) = G_TRUNC %10(<3 x i32>)
+ %11:_(<3 x i16>) = G_SHL %0, %1(<3 x i16>)
+ %16:_(i16), %17:_(i16), %18:_(i16) = G_UNMERGE_VALUES %11(<3 x i16>)
+ %13:_(i32) = G_ANYEXT %16(i16)
+ %14:_(i32) = G_ANYEXT %17(i16)
+ %15:_(i32) = G_ANYEXT %18(i16)
+ $vgpr0 = COPY %13(i32)
+ $vgpr1 = COPY %14(i32)
+ $vgpr2 = COPY %15(i32)
SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
...
@@ -175,87 +175,138 @@ body: |
; GFX7-LABEL: name: fma_v4f16
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
- ; GFX7-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
- ; GFX7-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
- ; GFX7-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
- ; GFX7-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
- ; GFX7-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
- ; GFX7-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY4]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
- ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY6]](s32)
- ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[COPY7]](s32)
- ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
- ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
- ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
- ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[COPY11]](s32)
- ; GFX7-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX7-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
- ; GFX7-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC8]](s16)
- ; GFX7-NEXT: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
- ; GFX7-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
- ; GFX7-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
- ; GFX7-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
- ; GFX7-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC9]](s16)
- ; GFX7-NEXT: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
- ; GFX7-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
- ; GFX7-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
- ; GFX7-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC6]](s16)
- ; GFX7-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC10]](s16)
- ; GFX7-NEXT: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
- ; GFX7-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
- ; GFX7-NEXT: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
- ; GFX7-NEXT: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
- ; GFX7-NEXT: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC11]](s16)
- ; GFX7-NEXT: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
- ; GFX7-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
- ; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
- ; GFX7-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
- ; GFX7-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC2]](s16)
- ; GFX7-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC3]](s16)
- ; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
- ; GFX7-NEXT: $vgpr1 = COPY [[ANYEXT1]](s32)
- ; GFX7-NEXT: $vgpr2 = COPY [[ANYEXT2]](s32)
- ; GFX7-NEXT: $vgpr3 = COPY [[ANYEXT3]](s32)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX7-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32)
+ ; GFX7-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32)
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR]](<2 x i32>)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR1]](<2 x i32>)
+ ; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i16>) = G_CONCAT_VECTORS [[TRUNC]](<2 x i16>), [[TRUNC1]](<2 x i16>)
+ ; GFX7-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x f16>) = G_BITCAST [[CONCAT_VECTORS]](<4 x i16>)
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
+ ; GFX7-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
+ ; GFX7-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr7
+ ; GFX7-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY4]](i32), [[COPY5]](i32)
+ ; GFX7-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR2]](<2 x i32>)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR3]](<2 x i32>)
+ ; GFX7-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x i16>) = G_CONCAT_VECTORS [[TRUNC2]](<2 x i16>), [[TRUNC3]](<2 x i16>)
+ ; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(<4 x f16>) = G_BITCAST [[CONCAT_VECTORS1]](<4 x i16>)
+ ; GFX7-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr8
+ ; GFX7-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr9
+ ; GFX7-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr10
+ ; GFX7-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr11
+ ; GFX7-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX7-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY10]](i32), [[COPY11]](i32)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR4]](<2 x i32>)
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(<2 x i16>) = G_TRUNC [[BUILD_VECTOR5]](<2 x i32>)
+ ; GFX7-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x i16>) = G_CONCAT_VECTORS [[TRUNC4]](<2 x i16>), [[TRUNC5]](<2 x i16>)
+ ; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(<4 x f16>) = G_BITCAST [[CONCAT_VECTORS2]](<4 x i16>)
+ ; GFX7-NEXT: [[UV:%[0-9]+]]:_(<2 x f16>), [[UV1:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BITCAST]](<4 x f16>)
+ ; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST3]](i32)
+ ; GFX7-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX7-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR]](i32)
+ ; GFX7-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST4]](i32)
+ ; GFX7-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX7-NEXT: [[UV2:%[0-9]+]]:_(<2 x f16>), [[UV3:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BITCAST1]](<4 x f16>)
+ ; GFX7-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC10:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST5]](i32)
+ ; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST5]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC11:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC12:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST6]](i32)
+ ; GFX7-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC13:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR3]](i32)
+ ; GFX7-NEXT: [[UV4:%[0-9]+]]:_(<2 x f16>), [[UV5:%[0-9]+]]:_(<2 x f16>) = G_UNMERGE_VALUES [[BITCAST2]](<4 x f16>)
+ ; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC14:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST7]](i32)
+ ; GFX7-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST7]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC15:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR4]](i32)
+ ; GFX7-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x f16>)
+ ; GFX7-NEXT: [[TRUNC16:%[0-9]+]]:_(f16) = G_TRUNC [[BITCAST8]](i32)
+ ; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST8]], [[C]](i32)
+ ; GFX7-NEXT: [[TRUNC17:%[0-9]+]]:_(f16) = G_TRUNC [[LSHR5]](i32)
+ ; GFX7-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC6]](f16)
+ ; GFX7-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC10]](f16)
+ ; GFX7-NEXT: [[FPEXT2:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC14]](f16)
+ ; GFX7-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+ ; GFX7-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMA]](f32)
+ ; GFX7-NEXT: [[FPEXT3:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC7]](f16)
+ ; GFX7-NEXT: [[FPEXT4:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC11]](f16)
+ ; GFX7-NEXT: [[FPEXT5:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC15]](f16)
+ ; GFX7-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
+ ; GFX7-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMA1]](f32)
+ ; GFX7-NEXT: [[FPEXT6:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC8]](f16)
+ ; GFX7-NEXT: [[FPEXT7:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC12]](f16)
+ ; GFX7-NEXT: [[FPEXT8:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC16]](f16)
+ ; GFX7-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
+ ; GFX7-NEXT: [[FPTRUNC2:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMA2]](f32)
+ ; GFX7-NEXT: [[FPEXT9:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC9]](f16)
+ ; GFX7-NEXT: [[FPEXT10:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC13]](f16)
+ ; GFX7-NEXT: [[FPEXT11:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC17]](f16)
+ ; GFX7-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
+ ; GFX7-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMA3]](f32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](f16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](f16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
+ ; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
+ ; GFX7-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
+ ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](f16)
+ ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](f16)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
+ ; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
+ ; GFX7-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
+ ; GFX7-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST9]](<2 x f16>), [[BITCAST10]](<2 x f16>)
+ ; GFX7-NEXT: [[BITCAST11:%[0-9]+]]:_(<4 x i16>) = G_BITCAST [[CONCAT_VECTORS3]](<4 x f16>)
+ ; GFX7-NEXT: [[UV6:%[0-9]+]]:_(<2 x i16>), [[UV7:%[0-9]+]]:_(<2 x i16>) = G_UNMERGE_VALUES [[BITCAST11]](<4 x i16>)
+ ; GFX7-NEXT: [[BITCAST12:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x i16>)
+ ; GFX7-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST12]], [[C]](i32)
+ ; GFX7-NEXT: [[BITCAST13:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x i16>)
+ ; GFX7-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST13]], [[C]](i32)
+ ; GFX7-NEXT: $vgpr0 = COPY [[BITCAST12]](i32)
+ ; GFX7-NEXT: $vgpr1 = COPY [[LSHR6]](i32)
+ ; GFX7-NEXT: $vgpr2 = COPY [[BITCAST13]](i32)
+ ; GFX7-NEXT: $vgpr3 = COPY [[LSHR7]](i32)
; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
- %4:_(s32) = COPY $vgpr0
- %5:_(s32) = COPY $vgpr1
- %6:_(s32) = COPY $vgpr2
- %7:_(s32) = COPY $vgpr3
- %8:_(<4 x s32>) = G_BUILD_VECTOR %4(s32), %5(s32), %6(s32), %7(s32)
- %0:_(<4 x s16>) = G_TRUNC %8(<4 x s32>)
- %9:_(s32) = COPY $vgpr4
- %10:_(s32) = COPY $vgpr5
- %11:_(s32) = COPY $vgpr6
- %12:_(s32) = COPY $vgpr7
- %13:_(<4 x s32>) = G_BUILD_VECTOR %9(s32), %10(s32), %11(s32), %12(s32)
- %1:_(<4 x s16>) = G_TRUNC %13(<4 x s32>)
- %14:_(s32) = COPY $vgpr8
- %15:_(s32) = COPY $vgpr9
- %16:_(s32) = COPY $vgpr10
- %17:_(s32) = COPY $vgpr11
- %18:_(<4 x s32>) = G_BUILD_VECTOR %14(s32), %15(s32), %16(s32), %17(s32)
- %2:_(<4 x s16>) = G_TRUNC %18(<4 x s32>)
- %19:_(<4 x s16>) = G_FMA %0, %1, %2
- %25:_(s16), %26:_(s16), %27:_(s16), %28:_(s16) = G_UNMERGE_VALUES %19(<4 x s16>)
- %21:_(s32) = G_ANYEXT %25(s16)
- %22:_(s32) = G_ANYEXT %26(s16)
- %23:_(s32) = G_ANYEXT %27(s16)
- %24:_(s32) = G_ANYEXT %28(s16)
- $vgpr0 = COPY %21(s32)
- $vgpr1 = COPY %22(s32)
- $vgpr2 = COPY %23(s32)
- $vgpr3 = COPY %24(s32)
+ %4:_(i32) = COPY $vgpr0
+ %5:_(i32) = COPY $vgpr1
+ %6:_(i32) = COPY $vgpr2
+ %7:_(i32) = COPY $vgpr3
+ %8:_(<4 x i32>) = G_BUILD_VECTOR %4(i32), %5(i32), %6(i32), %7(i32)
+ %0:_(<4 x i16>) = G_TRUNC %8(<4 x i32>)
+ %29:_(<4 x f16>) = G_BITCAST %0(<4 x i16>)
+ %9:_(i32) = COPY $vgpr4
+ %10:_(i32) = COPY $vgpr5
+ %11:_(i32) = COPY $vgpr6
+ %12:_(i32) = COPY $vgpr7
+ %13:_(<4 x i32>) = G_BUILD_VECTOR %9(i32), %10(i32), %11(i32), %12(i32)
+ %1:_(<4 x i16>) = G_TRUNC %13(<4 x i32>)
+ %30:_(<4 x f16>) = G_BITCAST %1(<4 x i16>)
+ %14:_(i32) = COPY $vgpr8
+ %15:_(i32) = COPY $vgpr9
+ %16:_(i32) = COPY $vgpr10
+ %17:_(i32) = COPY $vgpr11
+ %18:_(<4 x i32>) = G_BUILD_VECTOR %14(i32), %15(i32), %16(i32), %17(i32)
+ %2:_(<4 x i16>) = G_TRUNC %18(<4 x i32>)
+ %31:_(<4 x f16>) = G_BITCAST %2(<4 x i16>)
+ %19:_(<4 x f16>) = G_FMA %29, %30, %31
+ %32:_(<4 x i16>) = G_BITCAST %19(<4 x f16>)
+ %25:_(i16), %26:_(i16), %27:_(i16), %28:_(i16) = G_UNMERGE_VALUES %32(<4 x i16>)
+ %21:_(i32) = G_ANYEXT %25(i16)
+ %22:_(i32) = G_ANYEXT %26(i16)
+ %23:_(i32) = G_ANYEXT %27(i16)
+ %24:_(i32) = G_ANYEXT %28(i16)
+ $vgpr0 = COPY %21(i32)
+ $vgpr1 = COPY %22(i32)
+ $vgpr2 = COPY %23(i32)
+ $vgpr3 = COPY %24(i32)
SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
...
@@ -268,82 +319,85 @@ body: |
; GFX7-LABEL: name: maxnum_v5i16
; GFX7: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
; GFX7-NEXT: {{ $}}
- ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
- ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
- ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
- ; GFX7-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
- ; GFX7-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
- ; GFX7-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
- ; GFX7-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
- ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
- ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
- ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
- ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
- ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY4]](s32)
- ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
- ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY6]](s32)
- ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[COPY7]](s32)
- ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
- ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
- ; GFX7-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
- ; GFX7-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
- ; GFX7-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
- ; GFX7-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE]](s32)
- ; GFX7-NEXT: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
- ; GFX7-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC6]](s16)
- ; GFX7-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT2]], [[FPEXT3]]
- ; GFX7-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE1]](s32)
- ; GFX7-NEXT: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
- ; GFX7-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
- ; GFX7-NEXT: [[FMAXNUM_IEEE2:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT4]], [[FPEXT5]]
- ; GFX7-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE2]](s32)
- ; GFX7-NEXT: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
- ; GFX7-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC8]](s16)
- ; GFX7-NEXT: [[FMAXNUM_IEEE3:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT6]], [[FPEXT7]]
- ; GFX7-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE3]](s32)
- ; GFX7-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC4]](s16)
- ; GFX7-NEXT: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC9]](s16)
- ; GFX7-NEXT: [[FMAXNUM_IEEE4:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT8]], [[FPEXT9]]
- ; GFX7-NEXT: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE4]](s32)
- ; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
- ; GFX7-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
- ; GFX7-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC2]](s16)
- ; GFX7-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC3]](s16)
- ; GFX7-NEXT: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC4]](s16)
- ; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
- ; GFX7-NEXT: $vgpr1 = COPY [[ANYEXT1]](s32)
- ; GFX7-NEXT: $vgpr2 = COPY [[ANYEXT2]](s32)
- ; GFX7-NEXT: $vgpr3 = COPY [[ANYEXT3]](s32)
- ; GFX7-NEXT: $vgpr4 = COPY [[ANYEXT4]](s32)
+ ; GFX7-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX7-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX7-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
+ ; GFX7-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
+ ; GFX7-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
+ ; GFX7-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY]](i32)
+ ; GFX7-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY1]](i32)
+ ; GFX7-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY2]](i32)
+ ; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY3]](i32)
+ ; GFX7-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY4]](i32)
+ ; GFX7-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
+ ; GFX7-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
+ ; GFX7-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr7
+ ; GFX7-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr8
+ ; GFX7-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr9
+ ; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY5]](i32)
+ ; GFX7-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY6]](i32)
+ ; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[COPY7]](i32)
+ ; GFX7-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[COPY8]](i32)
+ ; GFX7-NEXT: [[TRUNC9:%[0-9]+]]:_(f16) = G_TRUNC [[COPY9]](i32)
+ ; GFX7-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC]](f16)
+ ; GFX7-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC5]](f16)
+ ; GFX7-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(f32) = G_FMAXNUM_IEEE [[FPEXT]], [[FPEXT1]]
+ ; GFX7-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMAXNUM_IEEE]](f32)
+ ; GFX7-NEXT: [[FPEXT2:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC1]](f16)
+ ; GFX7-NEXT: [[FPEXT3:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC6]](f16)
+ ; GFX7-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(f32) = G_FMAXNUM_IEEE [[FPEXT2]], [[FPEXT3]]
+ ; GFX7-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMAXNUM_IEEE1]](f32)
+ ; GFX7-NEXT: [[FPEXT4:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC2]](f16)
+ ; GFX7-NEXT: [[FPEXT5:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC7]](f16)
+ ; GFX7-NEXT: [[FMAXNUM_IEEE2:%[0-9]+]]:_(f32) = G_FMAXNUM_IEEE [[FPEXT4]], [[FPEXT5]]
+ ; GFX7-NEXT: [[FPTRUNC2:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMAXNUM_IEEE2]](f32)
+ ; GFX7-NEXT: [[FPEXT6:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC3]](f16)
+ ; GFX7-NEXT: [[FPEXT7:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC8]](f16)
+ ; GFX7-NEXT: [[FMAXNUM_IEEE3:%[0-9]+]]:_(f32) = G_FMAXNUM_IEEE [[FPEXT6]], [[FPEXT7]]
+ ; GFX7-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMAXNUM_IEEE3]](f32)
+ ; GFX7-NEXT: [[FPEXT8:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC4]](f16)
+ ; GFX7-NEXT: [[FPEXT9:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC9]](f16)
+ ; GFX7-NEXT: [[FMAXNUM_IEEE4:%[0-9]+]]:_(f32) = G_FMAXNUM_IEEE [[FPEXT8]], [[FPEXT9]]
+ ; GFX7-NEXT: [[FPTRUNC4:%[0-9]+]]:_(f16) = G_FPTRUNC [[FMAXNUM_IEEE4]](f32)
+ ; GFX7-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[FPTRUNC]](f16)
+ ; GFX7-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[FPTRUNC1]](f16)
+ ; GFX7-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[FPTRUNC2]](f16)
+ ; GFX7-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[FPTRUNC3]](f16)
+ ; GFX7-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[FPTRUNC4]](f16)
+ ; GFX7-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
+ ; GFX7-NEXT: $vgpr1 = COPY [[ANYEXT1]](i32)
+ ; GFX7-NEXT: $vgpr2 = COPY [[ANYEXT2]](i32)
+ ; GFX7-NEXT: $vgpr3 = COPY [[ANYEXT3]](i32)
+ ; GFX7-NEXT: $vgpr4 = COPY [[ANYEXT4]](i32)
; GFX7-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4
- %2:_(s32) = COPY $vgpr0
- %3:_(s32) = COPY $vgpr1
- %4:_(s32) = COPY $vgpr2
- %5:_(s32) = COPY $vgpr3
- %6:_(s32) = COPY $vgpr4
- %7:_(<5 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %4(s32), %5(s32), %6(s32)
- %0:_(<5 x s16>) = G_TRUNC %7(<5 x s32>)
- %8:_(s32) = COPY $vgpr5
- %9:_(s32) = COPY $vgpr6
- %10:_(s32) = COPY $vgpr7
- %11:_(s32) = COPY $vgpr8
- %12:_(s32) = COPY $vgpr9
- %13:_(<5 x s32>) = G_BUILD_VECTOR %8(s32), %9(s32), %10(s32), %11(s32), %12(s32)
- %1:_(<5 x s16>) = G_TRUNC %13(<5 x s32>)
- %15:_(<5 x s16>) = G_FMAXNUM %0, %1
- %21:_(s16), %22:_(s16), %23:_(s16), %24:_(s16), %25:_(s16) = G_UNMERGE_VALUES %15(<5 x s16>)
- %16:_(s32) = G_ANYEXT %21(s16)
- %17:_(s32) = G_ANYEXT %22(s16)
- %18:_(s32) = G_ANYEXT %23(s16)
- %19:_(s32) = G_ANYEXT %24(s16)
- %20:_(s32) = G_ANYEXT %25(s16)
- $vgpr0 = COPY %16(s32)
- $vgpr1 = COPY %17(s32)
- $vgpr2 = COPY %18(s32)
- $vgpr3 = COPY %19(s32)
- $vgpr4 = COPY %20(s32)
+ %2:_(i32) = COPY $vgpr0
+ %3:_(i32) = COPY $vgpr1
+ %4:_(i32) = COPY $vgpr2
+ %5:_(i32) = COPY $vgpr3
+ %6:_(i32) = COPY $vgpr4
+ %7:_(<5 x i32>) = G_BUILD_VECTOR %2(i32), %3(i32), %4(i32), %5(i32), %6(i32)
+ %0:_(<5 x i16>) = G_TRUNC %7(<5 x i32>)
+ %26:_(<5 x f16>) = G_BITCAST %0(<5 x i16>)
+ %8:_(i32) = COPY $vgpr5
+ %9:_(i32) = COPY $vgpr6
+ %10:_(i32) = COPY $vgpr7
+ %11:_(i32) = COPY $vgpr8
+ %12:_(i32) = COPY $vgpr9
+ %13:_(<5 x i32>) = G_BUILD_VECTOR %8(i32), %9(i32), %10(i32), %11(i32), %12(i32)
+ %1:_(<5 x i16>) = G_TRUNC %13(<5 x i32>)
+ %27:_(<5 x f16>) = G_BITCAST %1(<5 x i16>)
+ %15:_(<5 x f16>) = G_FMAXNUM %26, %27
+ %28:_(<5 x i16>) = G_BITCAST %15(<5 x f16>)
+ %21:_(i16), %22:_(i16), %23:_(i16), %24:_(i16), %25:_(i16) = G_UNMERGE_VALUES %28(<5 x i16>)
+ %16:_(i32) = G_ANYEXT %21(i16)
+ %17:_(i32) = G_ANYEXT %22(i16)
+ %18:_(i32) = G_ANYEXT %23(i16)
+ %19:_(i32) = G_ANYEXT %24(i16)
+ %20:_(i32) = G_ANYEXT %25(i16)
+ $vgpr0 = COPY %16(i32)
+ $vgpr1 = COPY %17(i32)
+ $vgpr2 = COPY %18(i32)
+ $vgpr3 = COPY %19(i32)
+ $vgpr4 = COPY %20(i32)
SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4
...
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index 7dd05ff26717a..e1c26a02f9a5d 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1,14 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
define float @fmul_select_f32_test1(float %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX7-LABEL: fmul_select_f32_test1:
@@ -2735,50 +2735,104 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
}
define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test1:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test1:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test1:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x3f80
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x4000
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test1:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x4000
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test1:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0x4000
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x3f80
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test1:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x3f80
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x4000
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test1:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x3f80
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test1:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x4000
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x3f80
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test1:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x3f80
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test1:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2821,46 +2875,25 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test1:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test1:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 2.000000e+00, bfloat 1.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -2868,50 +2901,104 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test2:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test2:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test2:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x3f80
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x3f00
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test2:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x3f00
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test2:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0x3f00
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x3f80
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test2:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x3f80
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x3f00
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test2:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x3f00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x3f80
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test2:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3f00
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x3f80
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test2:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x3f80
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3f00, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test2:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -2954,46 +3041,25 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test2:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test2:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x3f00 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3f80, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test2:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3f00, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 5.000000e-01, bfloat 1.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3001,76 +3067,168 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-LABEL: fmul_select_v2bf16_test3:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-NEXT: v_cndmask_b32_e64 v2, 1.0, 2.0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_v2bf16_test3:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, 1.0, 2.0, vcc
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, 1.0, 2.0, vcc
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_v2bf16_test3:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x3f80
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x4000
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
-; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
-; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_v2bf16_test3:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v6, 0x4000
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v7, 0x3f80
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v0
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v8, 0x7fff
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0x7fff, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_v2bf16_test3:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x4000
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x3f80
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v3, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_v2bf16_test3:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v3
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x3f80
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x4000
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v3, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_mov_b64 vcc, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-SDAG-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
+; GFX9-SDAG-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-SDAG-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_v2bf16_test3:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x4000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, 0x3f80
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v8, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v3, v3, v0, v8
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX9-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-GISEL-NEXT: v_add3_u32 v3, v3, v1, v8
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-GISEL-NEXT: v_perm_b32 v0, v1, v0, s4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_v2bf16_test3:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x4000
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e64 s4, v1, v3
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v3, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: s_mov_b32 vcc_lo, s4
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX10-SDAG-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX10-SDAG-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_v2bf16_test3:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0x3f80
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v5, 0x4000, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, v5, 0x4000, vcc_lo
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-GISEL-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v1, v3, v2
+; GFX10-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-GISEL-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX10-GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3142,30 +3300,31 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x3f80
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x4000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x4000, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -3177,33 +3336,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v5, 0x4000
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v5, 0x3f80
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x4000, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x4000, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v6
; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v2
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
%y = select <2 x i1> %bool, <2 x bfloat> <bfloat 2.000000e+00, bfloat 2.000000e+00>, <2 x bfloat> <bfloat 1.000000e+00, bfloat 1.000000e+00>
@@ -3212,87 +3372,179 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
}
define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-LABEL: fmul_select_v2bf16_test4:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-NEXT: v_cndmask_b32_e64 v2, 1.0, 0.5, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX7-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_v2bf16_test4:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, 1.0, 0.5, vcc
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, 1.0, 0.5, vcc
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_v2bf16_test4:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x3f80
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x3f00
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
-; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
-; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_v2bf16_test4:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v6, 0x3f00
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v7, 0x3f80
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v0
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v8, 0x7fff
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0x7fff, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_v2bf16_test4:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x3f00
-; GFX10-NEXT: v_mov_b32_e32 v4, 0x3f80
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v3, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_v2bf16_test4:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], v1, v3
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x3f80
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x3f00
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v3, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_mov_b64 vcc, s[4:5]
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-SDAG-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
+; GFX9-SDAG-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-SDAG-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-SDAG-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test4:
-; GFX11-SDAG-TRUE16: ; %bb.0:
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX9-GISEL-LABEL: fmul_select_v2bf16_test4:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v6, 0x3f00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v7, 0x3f80
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v8, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v3, v3, v0, v8
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX9-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-GISEL-NEXT: v_add3_u32 v3, v3, v1, v8
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-GISEL-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-GISEL-NEXT: v_perm_b32 v0, v1, v0, s4
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_v2bf16_test4:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x3f00
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e64 s4, v1, v3
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v3, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: s_mov_b32 vcc_lo, s4
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX10-SDAG-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX10-SDAG-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_v2bf16_test4:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 0x3f80
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v5, 0x3f00, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, v5, 0x3f00, vcc_lo
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-GISEL-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v1, v3, v2
+; GFX10-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-GISEL-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX10-GISEL-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test4:
+; GFX11-SDAG-TRUE16: ; %bb.0:
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
@@ -3353,30 +3605,31 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test4:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x3f80
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x3f00, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x3f00, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
@@ -3388,33 +3641,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test4:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v5, 0x3f00
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v5, 0x3f80
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x3f80, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x3f00, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v2, 0x3f80, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x3f00, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v6
; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, v3, v2
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq <2 x i32> %bool.arg1, %bool.arg2
%y = select <2 x i1> %bool, <2 x bfloat> <bfloat 5.000000e-01, bfloat 5.000000e-01>, <2 x bfloat> <bfloat 1.000000e+00, bfloat 1.000000e+00>
@@ -3423,51 +3677,105 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
}
define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test5:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, 0x41000000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v3, 2.0, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test5:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v3, 0x41000000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v3, 2.0, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test5:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x4100
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x4000
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test5:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x4000
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0x4100
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test5:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0x4000
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x4100
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test5:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x4100
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x4000
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test5:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4100
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test5:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x4000
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x4100
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test5:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x4100
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test5:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3510,46 +3818,25 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test5:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test5:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4100, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test5:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 2.000000e+00, bfloat 8.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3557,52 +3844,106 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test6:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, 0x40400000
-; GFX7-NEXT: v_mov_b32_e32 v4, 0xc1000000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test6:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v3, 0x40400000
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v4, 0xc1000000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test6:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x4040
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffffc100
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test6:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0xc100
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0x4040
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test6:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xffffc100
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x4040
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test6:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x4040
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffc100
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test6:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xc100
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4040
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test6:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffc100
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x4040
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test6:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x4040
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3645,46 +3986,25 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test6:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test6:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xffffc100 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4040, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test6:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4040 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -8.000000e+00, bfloat 3.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3692,51 +4012,105 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test7:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, 0x41000000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, -4.0, v3, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test7:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v3, 0x41000000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, -4.0, v3, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test7:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xffffc080
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x4100
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test7:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x4100
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0xc080
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test7:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0x4100
-; GFX10-NEXT: v_mov_b32_e32 v2, 0xffffc080
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test7:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffc080
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x4100
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test7:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4100
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xc080
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test7:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x4100
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffc080
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test7:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xc080
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3779,46 +4153,25 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test7:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test7:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xffffc080, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test7:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xc080 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 8.000000e+00, bfloat -4.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3826,50 +4179,102 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test8:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test8:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test8:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test8:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x8000
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test8:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 0xffff8000, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test8:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0xffff8000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test8:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test8:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0xffff8000, vcc_lo
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test8:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test8:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -3909,43 +4314,24 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test8:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test8:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffff8000, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test8:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -0.000000e+00, bfloat 0.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3953,52 +4339,106 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test9:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, 0xc2000000
-; GFX7-NEXT: v_mov_b32_e32 v4, 0xc1800000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test9:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v3, 0xc2000000
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v4, 0xc1800000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test9:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xffffc200
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffffc180
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test9:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0xc180
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0xc200
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test9:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xffffc180
-; GFX10-NEXT: v_mov_b32_e32 v2, 0xffffc200
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test9:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffc200
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffc180
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test9:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xc180
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xc200
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test9:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffc180
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffc200
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test9:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xc200
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc180, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test9:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -4041,46 +4481,25 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test9:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test9:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xffffc180 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xffffc200, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test9:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xc200 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc180, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -1.600000e+01, bfloat -3.200000e+01
%ldexp = fmul bfloat %x, %y
@@ -4088,52 +4507,106 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
}
define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, 0xdb800000
-; GFX7-NEXT: v_bfrev_b32_e32 v4, 7
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v3, 0xdb800000
+; GFX7-SDAG-NEXT: v_bfrev_b32_e32 v4, 7
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xffffdb80
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffffe000
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0xe000
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0xdb80
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xffffe000
-; GFX10-NEXT: v_mov_b32_e32 v2, 0xffffdb80
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffdb80
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffe000
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xe000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xdb80
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0xffffe000
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffdb80
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xdb80
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xe000, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -4176,46 +4649,25 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xffffe000 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xffffdb80, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xdb80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xe000, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 0xRE000, bfloat 0xRDB80
%ldexp = fmul bfloat %x, %y
@@ -4223,52 +4675,106 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
}
define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) {
-; GFX7-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfrev_b32_e32 v3, 50
-; GFX7-NEXT: v_mov_b32_e32 v4, 0x34800000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfrev_b32_e32 v3, 50
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v4, 0x34800000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0x4c00
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x3480
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
-; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, 0x3480
+; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, 0x4c00
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v0
+; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x7fff, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX7-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX7-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0x3480
-; GFX10-NEXT: v_mov_b32_e32 v2, 0x4c00
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX9-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x4c00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x3480
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x3480
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4c00
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-GISEL-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0x3480
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0x4c00
+; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-SDAG-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x4c00
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3480, vcc_lo
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX10-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
; GFX11-SDAG-TRUE16: ; %bb.0:
@@ -4311,49 +4817,36 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x3480 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4c00, v3, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4c00 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3480, vcc_lo
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 0xR3480, bfloat 0xR4C00
%ldexp = fmul bfloat %x, %y
ret bfloat %ldexp
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX11: {{.*}}
+; GFX11-GISEL-FAKE16: {{.*}}
+; GFX11-GISEL-TRUE16: {{.*}}
+; GFX11-SDAG: {{.*}}
+; GFX7: {{.*}}
+; GFX9: {{.*}}
More information about the llvm-commits
mailing list