[llvm] [AMDGPU][GlobalISel] Legalize odd bf16 vectors (PR #226707)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 08:53:24 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/226707
>From d1378cdf80b8a11eea525eaf500eaf1f277fefb0 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Sat, 26 Sep 2026 09:22:32 -0500
Subject: [PATCH 1/3] Use clampMaxNumElementsStrict
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 12 +-
.../CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll | 25 ++
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 200 ++++++++++++++++
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 72 ++++++
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 224 ++++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 51 ++++
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 81 +++++++
.../CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll | 26 ++
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 156 ++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll | 28 +++
.../AMDGPU/GlobalISel/strict_fadd.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fma.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fmul.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fsub.bf16.ll | 14 ++
14 files changed, 922 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c922d14576af8..3ae2515b18945 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1059,14 +1059,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// V2BF16
if (ST.hasBF16PackedInsts()) {
- MinNumMaxNumIeee.legalFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
- MinNumMaxNum.customFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
+ MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ MinNumMaxNum.customFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
MinNumMaxNumIeee.scalarize(0);
@@ -1205,7 +1199,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16}).clampMaxNumElements(0, BF16, 2);
+ FSubActions.lowerFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
if (ST.hasAnyPackedFP32Ops())
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
index 07f4e11691999..f7e40011531fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
@@ -100,3 +100,28 @@ define amdgpu_ps <2 x bfloat> @fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fabs_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fabs_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.fabs.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 04488ee933b39..9339b26ed66a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -411,6 +411,87 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fadd_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fadd_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-LABEL: fadd_v4bf16_vv:
; GFX9: ; %bb.0:
@@ -512,3 +593,122 @@ define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
%result = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %result
}
+define amdgpu_ps <5 x bfloat> @fadd_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX9-LABEL: fadd_v5bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v8, v8, v3, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX9-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v5bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-NEXT: v_dual_add_f32 v6, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-NEXT: v_dual_cndmask_b32 v3, v9, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX12-NEXT: v_add_f32_e32 v7, v7, v8
+; GFX12-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v7
+; GFX12-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: v_add3_u32 v8, v9, v7, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v10, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v5bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v3
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v4
+; GFX1250-NEXT: v_pk_add_bf16 v2, v2, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <5 x bfloat> %a, %b
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 8e777f7b6e2a0..949999c21bdbf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -235,3 +235,75 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
declare bfloat @llvm.canonicalize.bf16(bfloat)
declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
+
+define amdgpu_ps <3 x bfloat> @fcanonicalize_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fcanonicalize_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v2, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v2, v2, v2
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_dual_cndmask_b32 v1, v5, v6 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.canonicalize.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index c7dc27aa16340..3481f5fc096fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -455,6 +455,94 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
+define amdgpu_ps <3 x bfloat> @fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
+; GFX9-LABEL: fma_v3bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_fma_f32 v2, v2, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v3bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX12-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX12-NEXT: v_fmac_f32_e32 v8, v6, v7
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v8
+; GFX12-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v0, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v3bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v3, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c)
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fma_v4bf16_vvv(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
; GFX9-LABEL: fma_v4bf16_vvv:
; GFX9: ; %bb.0:
@@ -568,3 +656,139 @@ define amdgpu_ps <4 x bfloat> @fma_v4bf16_vvv(<4 x bfloat> %a, <4 x bfloat> %b,
%result = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %result
}
+
+define amdgpu_ps <5 x bfloat> @fma_v5bf16_vvv(<5 x bfloat> %a, <5 x bfloat> %b, <5 x bfloat> %c) {
+; GFX9-LABEL: fma_v5bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_fma_f32 v0, v0, v3, v6
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v15, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX9-NEXT: v_fma_f32 v3, v3, v6, v9
+; GFX9-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v9, v9, v3, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX9-NEXT: v_fma_f32 v1, v1, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v7
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v12
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
+; GFX9-NEXT: v_fma_f32 v4, v4, v6, v7
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_fma_f32 v2, v2, v5, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v5bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_fmac_f32_e32 v6, v0, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_dual_fmac_f32 v12, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v7.h
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX12-NEXT: v_fmac_f32_e32 v5, v2, v4
+; GFX12-NEXT: v_fmac_f32_e32 v11, v9, v10
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v12
+; GFX12-NEXT: v_bfe_u32 v9, v11, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v11
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v11
+; GFX12-NEXT: v_fmac_f32_e32 v1, v0, v3
+; GFX12-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX12-NEXT: v_add3_u32 v9, v9, v11, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v9, v9, v10, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v8, v10, v6, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v10, v12, 16, 1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
+; GFX12-NEXT: v_add3_u32 v6, v10, v12, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v12
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v9.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v5bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v3, v6
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v4, v7
+; GFX1250-NEXT: v_pk_fma_bf16 v2, v2, v5, v8
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <5 x bfloat> @llvm.fma.v5bf16(<5 x bfloat> %a, <5 x bfloat> %b, <5 x bfloat> %c)
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index e4bb63fa786b9..f87116aad81ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -131,6 +131,57 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmin_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX9-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX12-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
+ ret <3 x bfloat> %result
+}
+
+define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmax_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX9-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX1250-LABEL: fmin_v4bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 9208c5c805460..1cd14c61e7f24 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -411,6 +411,87 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fmul_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmul_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_mul_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fmul_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-LABEL: fmul_v4bf16_vv:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
index 31d064ea933bd..d1fbfce1c8fc6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
@@ -202,3 +202,29 @@ define amdgpu_ps <2 x bfloat> @fneg_fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = fneg <2 x bfloat> %fabs
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <3 x bfloat> @fneg_v3bf16_v(<3 x bfloat> %a) {
+; GFX9-LABEL: fneg_v3bf16_v:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_v3bf16_v:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX12-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_v3bf16_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1250-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg <3 x bfloat> %a
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index 06c2495ab4bff..ad20a1ec9e330 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -684,3 +684,159 @@ define amdgpu_ps <2 x bfloat> @fptrunc_v2f32_to_v2bf16_v(<2 x float> %a) {
%result = fptrunc <2 x float> %a to <2 x bfloat>
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <3 x bfloat> @fptrunc_v3f32_to_v3bf16(<3 x float> %a) {
+; GFX9-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX9-NEXT: v_cvt_pk_bf16_f32 v1, v2, v2
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX1250-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX1250-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX1250-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX1250-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX1250-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX1250-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
+ %result = fptrunc <3 x float> %a to <3 x bfloat>
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
index c120f166b17db..a9fba2761f7fc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -70,6 +70,19 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
}
+define amdgpu_ps <3 x bfloat> @fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX1250-LABEL: fsub_v4bf16_vv:
; GFX1250: ; %bb.0:
@@ -83,3 +96,18 @@ define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
%result = fsub <4 x bfloat> %a, %b
ret <4 x bfloat> %result
}
+
+define amdgpu_ps <5 x bfloat> @fsub_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v5bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v2, v2, v5 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <5 x bfloat> %a, %b
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
index ae790fb363e33..603166cc8dc17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
@@ -70,3 +70,17 @@ define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vl(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fadd_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fadd_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fadd.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
index 6eda1d592f4af..f8e9cd21ff7ee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
@@ -71,3 +71,17 @@ define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vll(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
+; GFX1250-LABEL: strict_fma_v3bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v3, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
index 0fc97f7731158..9cc2fb662c2db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
@@ -70,3 +70,17 @@ define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vl(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fmul_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fmul_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_mul_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fmul.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
index af3f46e7b2394..6a75044c7a325 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
@@ -31,3 +31,17 @@ define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fsub_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fsub.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
>From 2a848cb33afdcea9905824b1983ec309723483f8 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 28 Sep 2026 08:57:57 -0500
Subject: [PATCH 2/3] Remove useless check lines
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 24 -------------------
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 6 -----
2 files changed, 30 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index f87116aad81ba..c8d13c3874518 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -132,18 +132,6 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
-; GFX9-LABEL: fmin_v3bf16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_min_num_bf16 v0, v0, v2
-; GFX9-NEXT: v_pk_min_num_bf16 v1, v1, v3
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: fmin_v3bf16_vv:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_pk_min_num_bf16 v0, v0, v2
-; GFX12-NEXT: v_pk_min_num_bf16 v1, v1, v3
-; GFX12-NEXT: ; return to shader part epilog
-;
; GFX1250-LABEL: fmin_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -158,18 +146,6 @@ define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
}
define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
-; GFX9-LABEL: fmax_v3bf16_vv:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_pk_max_num_bf16 v0, v0, v2
-; GFX9-NEXT: v_pk_max_num_bf16 v1, v1, v3
-; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX12-LABEL: fmax_v3bf16_vv:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_pk_max_num_bf16 v0, v0, v2
-; GFX12-NEXT: v_pk_max_num_bf16 v1, v1, v3
-; GFX12-NEXT: ; return to shader part epilog
-;
; GFX1250-LABEL: fmax_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index ad20a1ec9e330..1c4a9b928f75b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -686,12 +686,6 @@ define amdgpu_ps <2 x bfloat> @fptrunc_v2f32_to_v2bf16_v(<2 x float> %a) {
}
define amdgpu_ps <3 x bfloat> @fptrunc_v3f32_to_v3bf16(<3 x float> %a) {
-; GFX9-LABEL: fptrunc_v3f32_to_v3bf16:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
-; GFX9-NEXT: v_cvt_pk_bf16_f32 v1, v2, v2
-; GFX9-NEXT: ; return to shader part epilog
-;
; GFX11-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
>From 1cd063fa71320a8568bfa0c280441939305e8767 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Tue, 29 Sep 2026 10:53:04 -0500
Subject: [PATCH 3/3] Legalize bf16 fadd/fmul/fma/fsub/fcanonicalize
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 59 +-
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 21 +-
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 22 +-
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 27 +-
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 970 ++++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 21 +-
.../CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll | 592 ++++++++++-
.../AMDGPU/GlobalISel/legalize-fadd.mir | 32 +-
.../GlobalISel/legalize-fcanonicalize.mir | 27 +-
.../AMDGPU/GlobalISel/legalize-fma.mir | 37 +-
.../AMDGPU/GlobalISel/legalize-fmul.mir | 32 +-
.../AMDGPU/GlobalISel/strict_fadd.bf16.ll | 29 +-
.../AMDGPU/GlobalISel/strict_fma.bf16.ll | 30 +-
.../AMDGPU/GlobalISel/strict_fmul.bf16.ll | 29 +-
.../AMDGPU/GlobalISel/strict_fsub.bf16.ll | 29 +-
15 files changed, 1759 insertions(+), 198 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 3ae2515b18945..5bad08c8b427d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -997,15 +997,21 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
- FCanonicalizeActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16,
- 2);
- StrictFPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ // Promote scalar bf16 operations to v2bf16 (packed) operations
+ FPOpActions.customFor({BF16}).legalFor({V2BF16}).clampMaxNumElementsStrict(
+ 0, BF16, 2);
+ FCanonicalizeActions.customFor({BF16})
+ .legalFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ StrictFPOpActions.customFor({BF16})
+ .legalFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ } else {
+ FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
+ FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
+ StrictFPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
- FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
- FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
-
if (ST.hasAnyPackedFP32Ops()) {
FPOpActions.legalFor({V2F32});
FCanonicalizeActions.legalFor({V2F32});
@@ -1060,7 +1066,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// V2BF16
if (ST.hasBF16PackedInsts()) {
MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
- MinNumMaxNum.customFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ MinNumMaxNum.customFor({BF16})
+ .customFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ } else {
+ MinNumMaxNum.widenScalarFor({BF16}, changeElementTo(0, F32));
}
MinNumMaxNumIeee.scalarize(0);
@@ -1199,7 +1209,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ FSubActions.customFor({BF16}).lowerFor({V2BF16}).clampMaxNumElementsStrict(
+ 0, BF16, 2);
+ } else {
+ FSubActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
if (ST.hasAnyPackedFP32Ops())
@@ -2426,6 +2439,25 @@ bool AMDGPULegalizerInfo::legalizeCustom(
return legalizeTrap(Helper, MI);
case TargetOpcode::G_DEBUGTRAP:
return legalizeDebugTrap(MI, MRI, B);
+ case TargetOpcode::G_FADD:
+ case TargetOpcode::G_FMUL:
+ case TargetOpcode::G_FMA:
+ case TargetOpcode::G_FSUB:
+ case TargetOpcode::G_FCANONICALIZE:
+ case TargetOpcode::G_STRICT_FADD:
+ case TargetOpcode::G_STRICT_FMUL:
+ case TargetOpcode::G_STRICT_FMA:
+ case TargetOpcode::G_STRICT_FSUB: {
+ // Promote scalar bf16 to v2bf16 for packed instruction use
+ // Only applies to targets with BF16 packed instructions
+ Register Dst = MI.getOperand(0).getReg();
+ if (ST.hasBF16PackedInsts() && MRI.getType(Dst) == BF16) {
+ // Use LegalizerHelper to promote to v2bf16
+ return Helper.moreElementsVector(MI, 0, V2BF16) ==
+ LegalizerHelper::Legalized;
+ }
+ return false;
+ }
default:
return false;
}
@@ -3000,8 +3032,17 @@ bool AMDGPULegalizerInfo::legalizeFPTOI(MachineInstr &MI,
bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineFunction &MF = Helper.MIRBuilder.getMF();
+ MachineRegisterInfo &MRI = *Helper.MIRBuilder.getMRI();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ // Promote scalar bf16 to v2bf16 for packed instruction use
+ // Only applies to targets with BF16 packed instructions
+ Register Dst = MI.getOperand(0).getReg();
+ if (ST.hasBF16PackedInsts() && MRI.getType(Dst) == BF16) {
+ return Helper.moreElementsVector(MI, 0, V2BF16) ==
+ LegalizerHelper::Legalized;
+ }
+
// With ieee_mode disabled, the instructions have the correct behavior.
if (!MFI->getMode().IEEE)
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 9339b26ed66a8..2a9512b63752d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -37,14 +37,7 @@ define amdgpu_ps bfloat @fadd_bf16_vv(bfloat %a, bfloat %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
ret bfloat %result
@@ -90,16 +83,8 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_add_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 949999c21bdbf..c73fb6fdfd45f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -35,14 +35,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_v(bfloat %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
ret bfloat %result
@@ -91,16 +84,8 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, s0 op_sel_hi:[0,1]
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
@@ -233,9 +218,6 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
ret <2 x bfloat> %result
}
-declare bfloat @llvm.canonicalize.bf16(bfloat)
-declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
-
define amdgpu_ps <3 x bfloat> @fcanonicalize_v3bf16_vv(<3 x bfloat> %a) {
; GFX9-LABEL: fcanonicalize_v3bf16_vv:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index 3481f5fc096fa..9c5cee8ad890d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -39,15 +39,7 @@ define amdgpu_ps bfloat @fma_bf16_vvv(bfloat %a, bfloat %b, bfloat %c) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX1250-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
-; GFX1250-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %result
@@ -96,17 +88,9 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
-; GFX1250-NEXT: s_fmac_f32 s2, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s0, s2, 0x10010
-; GFX1250-NEXT: s_or_b32 s1, s2, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
-; GFX1250-NEXT: s_addk_co_i32 s0, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
@@ -452,9 +436,6 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
-declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
-declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
-
define amdgpu_ps <3 x bfloat> @fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
; GFX9-LABEL: fma_v3bf16_vvv:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index c8d13c3874518..644701a77b3af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -1,7 +1,233 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+define amdgpu_ps bfloat @fmin_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fmin_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmin_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fmin_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_min_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmax_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fmax_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmax_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fmax_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_max_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fmin_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_min_num_f32 v0, v0, v1 :: v_dual_min_num_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -15,6 +241,54 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmin_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -28,6 +302,66 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmin_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_min_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_min_num_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_min_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -41,6 +375,48 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fmin_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -54,6 +430,48 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fmin_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -67,6 +485,54 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fmax_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_max_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -80,6 +546,54 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmax_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -93,6 +607,66 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmax_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_max_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_max_num_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_max_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -106,6 +680,48 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fmax_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -119,6 +735,48 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fmax_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -132,6 +790,74 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmin_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_min_num_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_min_num_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -146,6 +872,74 @@ define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
}
define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmax_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_max_num_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -159,6 +953,94 @@ define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
ret <3 x bfloat> %result
}
define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fmin_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_min_num_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_min_num_f32 v1, v1, v3 :: v_dual_min_num_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -173,6 +1055,94 @@ define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
}
define amdgpu_ps <4 x bfloat> @fmax_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fmax_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_max_num_f32 v1, v1, v3 :: v_dual_max_num_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 1cd14c61e7f24..b21aa2055f0d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -37,14 +37,7 @@ define amdgpu_ps bfloat @fmul_bf16_vv(bfloat %a, bfloat %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
ret bfloat %result
@@ -90,16 +83,8 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_mul_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
index a9fba2761f7fc..6a8f53493cbf5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -1,10 +1,145 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
-; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @fsub_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fsub_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fsub_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fsub_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fsub_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_sub_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fsub_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub bfloat %a, %b
+ ret bfloat %result
+}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fsub_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_sub_f32 v0, v0, v1 :: v_dual_sub_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -18,6 +153,54 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fsub_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_subrev_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -31,6 +214,66 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fsub_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_sub_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_sub_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_sub_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -44,6 +287,48 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fsub_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_subrev_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -57,6 +342,48 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fsub_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_subrev_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -71,6 +398,74 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
define amdgpu_ps <3 x bfloat> @fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fsub_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_sub_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -84,6 +479,94 @@ define amdgpu_ps <3 x bfloat> @fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
ret <3 x bfloat> %result
}
define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fsub_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_sub_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_sub_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -98,6 +581,111 @@ define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
}
define amdgpu_ps <5 x bfloat> @fsub_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX9-LABEL: fsub_v5bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v8, v8, v3, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX9-NEXT: v_sub_f32_e32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v5bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_dual_sub_f32 v2, v2, v5 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX12-NEXT: v_dual_sub_f32 v6, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-NEXT: v_dual_cndmask_b32 v3, v9, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_sub_f32_e32 v1, v1, v4
+; GFX12-NEXT: v_sub_f32_e32 v7, v7, v8
+; GFX12-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v7
+; GFX12-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: v_add3_u32 v8, v9, v7, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v10, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v5bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 6702c344c979c..5a4b69b7ab82b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -929,32 +929,12 @@ body: |
; GFX1250-LABEL: name: test_fadd_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x bf16>) = G_FADD [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FADD]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_FADD %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index c6d7820a7dd28..93f8bc2fc173d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -645,28 +645,11 @@ body: |
;
; GFX1250-LABEL: name: test_fcanonicalize_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST1]]
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST3]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x bf16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FCANONICALIZE]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_FCANONICALIZE %0
S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index ca1ba32ef7424..a0dd4a4e4b89f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1090,36 +1090,13 @@ body: |
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF2:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF2]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST7]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF2]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x bf16>) = G_FMA [[BUILD_VECTOR]], [[BUILD_VECTOR1]], [[BUILD_VECTOR2]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FMA]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 1f01ecfe5fb77..3ba52a65e4364 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -844,32 +844,12 @@ body: |
; GFX1250-LABEL: name: test_fmul_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x bf16>) = G_FMUL [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FMUL]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_FMUL %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
index 603166cc8dc17..7bb90de39c526 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fadd_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fadd_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fadd_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fadd_v2bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
index f8e9cd21ff7ee..2de33c4d40d45 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
@@ -1,9 +1,37 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fma_bf16_vvv(bfloat %a, bfloat %b, bfloat %c) #0 {
+; GFX1250-LABEL: strict_fma_bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fma.bf16(bfloat %a, bfloat %b, bfloat %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat inreg %c) #0 {
+; GFX1250-LABEL: strict_fma_bf16_sss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fma.bf16(bfloat %a, bfloat %b, bfloat %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vvv(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
; GFX1250-LABEL: strict_fma_v2bf16_vvv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
index 9cc2fb662c2db..2fa7259411eb3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fmul_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fmul_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fmul.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fmul_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fmul.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fmul_v2bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
index 6a75044c7a325..c3b3740349660 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fsub_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fsub_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fsub.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fsub_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fsub_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fsub.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fsub_v2bf16_vv:
; GFX1250: ; %bb.0:
More information about the llvm-commits
mailing list