[llvm] cd6eae7 - [AMDGPU][GlobalISel] Legalize bf16 for fadd, fmul, fma, fsub, fmin, fmax, and fcanonicalize (#227901)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 7 06:50:29 PDT 2026
Author: LU-JOHN
Date: 2026-10-07T13:50:15Z
New Revision: cd6eae76f085bba5c703a4385b63788d391d3efb
URL: https://github.com/llvm/llvm-project/commit/cd6eae76f085bba5c703a4385b63788d391d3efb
DIFF: https://github.com/llvm/llvm-project/commit/cd6eae76f085bba5c703a4385b63788d391d3efb.diff
LOG: [AMDGPU][GlobalISel] Legalize bf16 for fadd, fmul, fma, fsub, fmin, fmax, and fcanonicalize (#227901)
Legalize bf16 operations for fadd, fmul, fma, fsub, fmin, fmax, and
fcanonicalize including strict variants.
Promote bf16 to v2bf16 for GFX1250+. Otherwise expand to f32.
Enable testing for GFX9, GFX12 for fmin, fmax, and fsub which was
blocked by a lack of bf16 support.
---------
Signed-off-by: John Lu <John.Lu at amd.com>
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 360ca956f22b5..0fefe3f6da884 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -993,15 +993,21 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
- FCanonicalizeActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16,
- 2);
- StrictFPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ // Promote scalar bf16 operations to v2bf16 (packed) operations
+ FPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
+ .legalFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ FCanonicalizeActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
+ .legalFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ StrictFPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
+ .legalFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ } else {
+ FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
+ FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
- FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
- FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
-
if (ST.hasAnyPackedFP32Ops()) {
FPOpActions.legalFor({V2F32});
FCanonicalizeActions.legalFor({V2F32});
@@ -1056,7 +1062,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// V2BF16
if (ST.hasBF16PackedInsts()) {
MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
- MinNumMaxNum.customFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ MinNumMaxNum.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
+ .customFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ } else {
+ MinNumMaxNum.widenScalarFor({BF16}, changeElementTo(0, F32));
}
MinNumMaxNumIeee.scalarize(0);
@@ -1195,7 +1205,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ FSubActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
+ .lowerFor({V2BF16})
+ .clampMaxNumElementsStrict(0, BF16, 2);
+ } else {
+ FSubActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
if (ST.hasAnyPackedFP32Ops())
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 479ac3421be70..ac8ad6dd7f67f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -37,14 +37,7 @@ define amdgpu_ps bfloat @fadd_bf16_vv(bfloat %a, bfloat %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
ret bfloat %result
@@ -90,16 +83,8 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_add_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 949999c21bdbf..c73fb6fdfd45f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -35,14 +35,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_v(bfloat %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
ret bfloat %result
@@ -91,16 +84,8 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_max_num_f32_e64 v0, s0, s0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, s0 op_sel_hi:[0,1]
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
@@ -233,9 +218,6 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
ret <2 x bfloat> %result
}
-declare bfloat @llvm.canonicalize.bf16(bfloat)
-declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
-
define amdgpu_ps <3 x bfloat> @fcanonicalize_v3bf16_vv(<3 x bfloat> %a) {
; GFX9-LABEL: fcanonicalize_v3bf16_vv:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index a3a116248a7fd..3902986b951f8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -39,15 +39,7 @@ define amdgpu_ps bfloat @fma_bf16_vvv(bfloat %a, bfloat %b, bfloat %c) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX1250-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
-; GFX1250-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %result
@@ -96,17 +88,9 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
-; GFX1250-NEXT: s_fmac_f32 s2, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s0, s2, 0x10010
-; GFX1250-NEXT: s_or_b32 s1, s2, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
-; GFX1250-NEXT: s_addk_co_i32 s0, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index cc819ab215ea7..987a68f7cab39 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -1,7 +1,233 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+define amdgpu_ps bfloat @fmin_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fmin_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmin_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fmin_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_min_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmax_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fmax_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fmax_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fmax_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_max_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fmin_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_min_num_f32 v0, v0, v1 :: v_dual_min_num_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -15,6 +241,54 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmin_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -28,6 +302,66 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmin_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_min_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_min_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_min_num_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_min_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -41,6 +375,48 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fmin_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -54,6 +430,48 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fmin_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_min_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_min_num_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -67,6 +485,54 @@ define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fmax_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_max_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -80,6 +546,54 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmax_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -93,6 +607,66 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fmax_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_max_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_max_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_max_num_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_max_num_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -106,6 +680,48 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fmax_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -119,6 +735,48 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fmax_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_max_num_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -132,6 +790,74 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmin_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_min_num_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_min_num_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -146,6 +872,74 @@ define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
}
define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmax_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_max_num_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -160,6 +954,94 @@ define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b)
}
define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fmin_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_min_num_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_min_num_f32 v1, v1, v3 :: v_dual_min_num_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmin_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -174,6 +1056,94 @@ define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
}
define amdgpu_ps <4 x bfloat> @fmax_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fmax_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_max_num_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_max_num_f32 v1, v1, v3 :: v_dual_max_num_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fmax_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index f0070d4a44ac0..ee789f78ec0b3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -37,14 +37,7 @@ define amdgpu_ps bfloat @fmul_bf16_vv(bfloat %a, bfloat %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
ret bfloat %result
@@ -90,16 +83,8 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_mul_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
index f957345c53a2e..42a266ad41312 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -1,10 +1,145 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
-; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @fsub_bf16_vv(bfloat %a, bfloat %b) {
+; GFX9-LABEL: fsub_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fsub_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub bfloat %a, %b
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fsub_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
+; GFX9-LABEL: fsub_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s2, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s2, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_sub_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fsub_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub bfloat %a, %b
+ ret bfloat %result
+}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX9-LABEL: fsub_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_dual_sub_f32 v0, v0, v1 :: v_dual_sub_f32 v1, v2, v3
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -18,6 +153,54 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fsub_v2bf16_vs:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_lshr_b32 s1, s0, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_subrev_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: s_lshl_b32 s0, s1, 16
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, s0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vs:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: s_lshl_b32 s1, s0, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, s0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, s1, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vs:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -31,6 +214,66 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX9-LABEL: fsub_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_lshr_b32 s3, s1, 16
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_sub_f32_e32 v0, s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_bfe_u32 s4, s0, 0x10010
+; GFX9-NEXT: s_or_b32 s1, s0, 0x400000
+; GFX9-NEXT: s_add_i32 s0, s4, s0
+; GFX9-NEXT: s_addk_i32 s0, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_lshl_b32 s1, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_sub_f32_e32 v0, s1, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_bfe_u32 s3, s1, 0x10010
+; GFX9-NEXT: s_or_b32 s2, s1, 0x400000
+; GFX9-NEXT: s_add_i32 s1, s3, s1
+; GFX9-NEXT: s_addk_i32 s1, 0x7fff
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s1, s2, s1
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_lshl_b32 s2, s0, 16
+; GFX12-NEXT: s_lshl_b32 s3, s1, 16
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_sub_f32 s2, s2, s3
+; GFX12-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10010
+; GFX12-NEXT: s_or_b32 s4, s2, 0x400000
+; GFX12-NEXT: s_add_co_i32 s3, s3, s2
+; GFX12-NEXT: s_addk_co_i32 s3, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-NEXT: s_lshl_b32 s0, s0, 16
+; GFX12-NEXT: s_lshl_b32 s1, s1, 16
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_sub_f32 s0, s0, s1
+; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT: s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-NEXT: s_cselect_b32 s0, s3, s1
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_ss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -44,6 +287,48 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
+; GFX9-LABEL: fsub_v2bf16_vc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_subrev_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, 2.0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vc:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, 2.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vc:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -57,6 +342,48 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
}
define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
+; GFX9-LABEL: fsub_v2bf16_vl:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_subrev_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_subrev_f32_e32 v1, 0x42c80000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v2bf16_vl:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_dual_subrev_f32 v1, 0x42c80000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_subrev_f32_e32 v0, 1.0, v0
+; GFX12-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v2bf16_vl:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -70,6 +397,74 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
}
define amdgpu_ps <3 x bfloat> @fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fsub_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_sub_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v3bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -98,6 +493,94 @@ define amdgpu_ps <3 x bfloat> @fsub_v3bf16_ss(<3 x bfloat> inreg %a, <3 x bfloat
}
define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fsub_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_sub_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_sub_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v4bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -126,6 +609,111 @@ define amdgpu_ps <4 x bfloat> @fsub_v4bf16_ss(<4 x bfloat> inreg %a, <4 x bfloat
}
define amdgpu_ps <5 x bfloat> @fsub_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX9-LABEL: fsub_v5bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v8, v8, v3, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_sub_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX9-NEXT: v_sub_f32_e32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_sub_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fsub_v5bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_dual_sub_f32 v2, v2, v5 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX12-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX12-NEXT: v_dual_sub_f32 v6, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-NEXT: v_dual_cndmask_b32 v3, v9, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_sub_f32_e32 v1, v1, v4
+; GFX12-NEXT: v_sub_f32_e32 v7, v7, v8
+; GFX12-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v7
+; GFX12-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: v_add3_u32 v8, v9, v7, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v10, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX12-NEXT: ; return to shader part epilog
+;
; GFX1250-LABEL: fsub_v5bf16_vv:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 6702c344c979c..5a4b69b7ab82b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -929,32 +929,12 @@ body: |
; GFX1250-LABEL: name: test_fadd_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x bf16>) = G_FADD [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FADD]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_FADD %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index c6d7820a7dd28..93f8bc2fc173d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -645,28 +645,11 @@ body: |
;
; GFX1250-LABEL: name: test_fcanonicalize_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST1]]
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST3]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x bf16>) = G_FCANONICALIZE [[BUILD_VECTOR]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FCANONICALIZE]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_FCANONICALIZE %0
S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index ca1ba32ef7424..a0dd4a4e4b89f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1090,36 +1090,13 @@ body: |
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF2:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF2]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST7]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF2]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x bf16>) = G_FMA [[BUILD_VECTOR]], [[BUILD_VECTOR1]], [[BUILD_VECTOR2]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FMA]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 1f01ecfe5fb77..3ba52a65e4364 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -844,32 +844,12 @@ body: |
; GFX1250-LABEL: name: test_fmul_bf16
; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BITCAST5]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[DEF1]](bf16), [[DEF]](bf16)
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x bf16>) = G_FMUL [[BUILD_VECTOR]], [[BUILD_VECTOR1]]
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[FMUL]](<2 x bf16>)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(bf16) = COPY [[UV]](bf16)
+ ; GFX1250-NEXT: S_NOP 0, implicit [[COPY]](bf16)
%0:_(bf16) = G_IMPLICIT_DEF
%1:_(bf16) = G_IMPLICIT_DEF
%2:_(bf16) = G_FMUL %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
index 6e241c6824e66..6ce9f7a1604ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fadd_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fadd_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fadd_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fadd_v2bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
index 1e78ba4366905..d2ef6c44f3a52 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
@@ -1,9 +1,37 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fma_bf16_vvv(bfloat %a, bfloat %b, bfloat %c) #0 {
+; GFX1250-LABEL: strict_fma_bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fma.bf16(bfloat %a, bfloat %b, bfloat %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat inreg %c) #0 {
+; GFX1250-LABEL: strict_fma_bf16_sss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fma.bf16(bfloat %a, bfloat %b, bfloat %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vvv(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
; GFX1250-LABEL: strict_fma_v2bf16_vvv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
index 2471905f40227..c68bc1d74e369 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fmul_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fmul_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fmul.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fmul_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fmul.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fmul_v2bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
index bd1c94ee0f7c2..26caab825bfb8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
@@ -1,9 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
-; TODO: Add scalar bf16 tests when scalar bf16 support is added.
; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+define amdgpu_ps bfloat @strict_fsub_bf16_vv(bfloat %a, bfloat %b) #0 {
+; GFX1250-LABEL: strict_fsub_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fsub.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @strict_fsub_bf16_ss(bfloat inreg %a, bfloat inreg %b) #0 {
+; GFX1250-LABEL: strict_fsub_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.experimental.constrained.fsub.bf16(bfloat %a, bfloat %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret bfloat %result
+}
+
define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX1250-LABEL: strict_fsub_v2bf16_vv:
; GFX1250: ; %bb.0:
More information about the llvm-commits
mailing list