[llvm] [AMDGPU][GISel] Legalize v2bf16 (PR #223803)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 14:49:46 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/223803
>From 0675589c978d42c4d46861991c1c6d12d2ec543d Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Fri, 11 Sep 2026 08:42:20 -0500
Subject: [PATCH 1/5] Legalize v2bf16
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 76 ++++++----
.../CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll | 102 ++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 91 +-----------
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 39 +-----
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 111 +--------------
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 132 ++++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 91 +-----------
.../CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll | 102 ++++++++++++++
.../AMDGPU/GlobalISel/legalize-fadd.mir | 51 +------
.../GlobalISel/legalize-fcanonicalize.mir | 43 +-----
.../AMDGPU/GlobalISel/legalize-fma.mir | 61 +-------
.../AMDGPU/GlobalISel/legalize-fmul.mir | 52 +------
12 files changed, 418 insertions(+), 533 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 6ca65e6fa01af..82576c4e2bd7b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -727,8 +727,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const std::initializer_list<LLT> FPTypesBase = {F32, F64};
const std::initializer_list<LLT> FPTypes16 = {F32, F64, F16};
const std::initializer_list<LLT> FPTypesPK16 = {F32, F64, F16, V2F16};
- const std::initializer_list<LLT> FPTypesPK16_64 = {F32, F64, F16, V2F16,
- V2F64};
const LLT I1 = LLT::integer(1);
const LLT I16 = LLT::integer(16);
@@ -998,6 +996,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
FDIVActions.customFor({F16});
}
+ if (ST.hasBF16PackedInsts()) {
+ FPOpActions.legalFor({V2BF16});
+ FCanonicalizeActions.legalFor({V2BF16});
+ StrictFPOpActions.legalFor({V2BF16});
+ }
+
FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
@@ -1021,38 +1025,52 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &MinNumMaxNumIeee =
getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
+ auto &MinNumMaxNum = getActionDefinitionsBuilder(
+ {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
- if (ST.hasVOP3PInsts()) {
- MinNumMaxNumIeee.legalFor(FPTypesPK16)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .clampMaxNumElements(0, F16, 2)
- .scalarize(0);
- } else if (ST.has16BitInsts()) {
- MinNumMaxNumIeee.legalFor(FPTypes16).scalarize(0);
- } else {
- MinNumMaxNumIeee.legalFor(FPTypesBase).scalarize(0);
+ MinNumMaxNumIeee.legalFor({F32, F64});
+ MinNumMaxNum.customFor({F32, F64});
+
+ if (ST.has16BitInsts()) {
+ MinNumMaxNumIeee.legalFor({F16});
+ MinNumMaxNum.customFor({F16});
}
- auto &MinNumMaxNum = getActionDefinitionsBuilder(
- {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
+ // V2F16
+ if (ST.hasVOP3PInsts()) {
+ MinNumMaxNumIeee.legalFor({V2F16})
+ .moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
+ oneMoreElement(0))
+ .clampMaxNumElements(0, F16, 2);
+ MinNumMaxNum.customFor({V2F16})
+ .moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
+ oneMoreElement(0))
+ .clampMaxNumElements(0, F16, 2);
+ }
+ // V2F64
if (ST.hasAnyPackedFP64Ops()) {
- MinNumMaxNum.customFor(FPTypesPK16_64)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .clampMaxNumElements(0, F16, 2)
- .clampMaxNumElements(0, F64, 2)
- .scalarize(0);
- } else if (ST.hasVOP3PInsts()) {
- MinNumMaxNum.customFor(FPTypesPK16)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .clampMaxNumElements(0, F16, 2)
- .scalarize(0);
- } else if (ST.has16BitInsts()) {
- MinNumMaxNum.customFor(FPTypes16).scalarize(0);
- } else {
- MinNumMaxNum.customFor(FPTypesBase).scalarize(0);
+ MinNumMaxNum.customFor({V2F64})
+ .moreElementsIf(all(elementTypeIs(0, F64), isSmallOddVector(0)),
+ oneMoreElement(0))
+ .clampMaxNumElements(0, F64, 2);
+ }
+
+ // V2BF16
+ if (ST.hasBF16PackedInsts()) {
+ MinNumMaxNumIeee.legalFor({V2BF16})
+ .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
+ oneMoreElement(0))
+ .clampMaxNumElements(0, BF16, 2);
+ MinNumMaxNum.customFor({V2BF16})
+ .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
+ oneMoreElement(0))
+ .clampMaxNumElements(0, BF16, 2);
}
+ MinNumMaxNumIeee.scalarize(0);
+ MinNumMaxNum.scalarize(0);
+
if (!ST.has16BitInsts()) {
MinNumMaxNumIeee.minScalar(0, F32);
MinNumMaxNum.minScalar(0, F32);
@@ -1185,6 +1203,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lowerFor({F64, F16, V2F16});
}
+ if (ST.hasBF16PackedInsts()) {
+ FSubActions.lowerFor({V2BF16});
+ }
+
if (ST.hasAnyPackedFP32Ops())
FSubActions.lowerFor({V2F32}).clampMaxNumElements(0, F32, 2);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
new file mode 100644
index 0000000000000..07f4e11691999
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fabs_bf16_vv(bfloat %a) {
+; GFX9-LABEL: fabs_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.fabs.bf16(bfloat %a)
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fabs_bf16_ss(bfloat inreg %a) {
+; GFX9-LABEL: fabs_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call bfloat @llvm.fabs.bf16(bfloat %a)
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fabs_v2bf16_vv(<2 x bfloat> %a) {
+; GFX9-LABEL: fabs_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
+; GFX9-LABEL: fabs_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_and_b32 s0, s0, 0x7fff7fff
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff7fff
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_and_b32 s0, s0, 0x7fff7fff
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
+ ret <2 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 512163fdc3272..38fd0cecbf90a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -161,22 +161,7 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX1250-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX1250-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_add_f32 v1, v2, v3
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -237,24 +222,7 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_lshl_b32 s1, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_dual_add_f32 v0, s1, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_add_f32_e32 v1, s0, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -327,30 +295,7 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s2, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s3, s1, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_add_f32 s2, s2, s3
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10010
-; GFX1250-NEXT: s_or_b32 s4, s2, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s3, s3, s2
-; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s2, s4, s3
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
-; GFX1250-NEXT: s_add_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s2, s0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -405,20 +350,7 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vc(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_add_f32 v0, 2.0, v0 :: v_dual_add_f32 v1, 2.0, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 2.0 op_sel:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
ret <2 x bfloat> %result
@@ -473,20 +405,7 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v1, 0x42c80000, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_add_bf16 v0, 0x42c83f80, v0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
ret <2 x bfloat> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index cc029aa82e8e3..8e777f7b6e2a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -156,20 +156,7 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_v(<2 x bfloat> %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_max_num_f32 v1, v1, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat> %src)
ret <2 x bfloat> %result
@@ -240,29 +227,7 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s1, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_max_num_f32_e64 v0, s1, s1
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1250-NEXT: s_bfe_u32 s2, s1, 0x10010
-; GFX1250-NEXT: s_or_b32 s3, s1, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s2, s2, s1
-; GFX1250-NEXT: s_addk_co_i32 s2, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s1, 0
-; GFX1250-NEXT: s_cselect_b32 s1, s3, s2
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: v_max_num_f32_e64 v0, s0, s0
-; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1250-NEXT: s_bfe_u32 s2, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s2, s2, s0
-; GFX1250-NEXT: s_addk_co_i32 s2, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s2
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, s0 op_sel_hi:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat> %src)
ret <2 x bfloat> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index 61bc3fe49a716..4401368f54421 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -174,25 +174,7 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vvv(<2 x bfloat> %a, <2 x bfloat> %b,
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX1250-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX1250-NEXT: v_mov_b16_e32 v5.l, v2.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX1250-NEXT: v_dual_fmac_f32 v2, v3, v1 :: v_dual_fmac_f32 v5, v0, v4
-; GFX1250-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
-; GFX1250-NEXT: v_bfe_u32 v1, v5, 16, 1
-; GFX1250-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v5
-; GFX1250-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v0, v3, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
ret <2 x bfloat> %result
@@ -262,28 +244,7 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vss(<2 x bfloat> %a, <2 x bfloat> inre
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_lshr_b32 s2, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s3, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
-; GFX1250-NEXT: v_fma_f32 v0, v0, s0, s1
-; GFX1250-NEXT: s_lshl_b32 s0, s3, 16
-; GFX1250-NEXT: v_fma_f32 v1, v1, s2, s0
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
ret <2 x bfloat> %result
@@ -364,33 +325,8 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_sss(<2 x bfloat> inreg %a, <2 x bfloat
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s3, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s4, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s5, s2, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_fmac_f32 s5, s3, s4
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
-; GFX1250-NEXT: s_bfe_u32 s3, s5, 0x10010
-; GFX1250-NEXT: s_or_b32 s4, s5, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s3, s3, s5
-; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s5, 0
-; GFX1250-NEXT: s_cselect_b32 s3, s4, s3
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
-; GFX1250-NEXT: s_fmac_f32 s2, s0, s1
-; GFX1250-NEXT: s_lshr_b32 s1, s3, 16
-; GFX1250-NEXT: s_bfe_u32 s0, s2, 0x10010
-; GFX1250-NEXT: s_or_b32 s3, s2, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
-; GFX1250-NEXT: s_addk_co_i32 s0, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s0
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
ret <2 x bfloat> %result
@@ -451,25 +387,7 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vsc(<2 x bfloat> %a, <2 x bfloat> inre
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_lshl_b32 s1, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX1250-NEXT: v_fma_f32 v0, v0, s1, 0.5
-; GFX1250-NEXT: v_fma_f32 v1, v1, s0, 0.5
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, s0, 0.5 op_sel:[0,0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)
ret <2 x bfloat> %result
@@ -527,23 +445,8 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: s_mov_b32 s0, 0x43480000
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT: v_fma_f32 v0, v0, 1.0, 2.0
-; GFX1250-NEXT: s_mov_b32 s0, 0x400000
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_bitop2_b32 v4, s0, v0 bitop3:0x54
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_fmac_f32_e32 v2, 0x42c80000, v1
-; GFX1250-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX1250-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX1250-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
-; GFX1250-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0x43484000
+; GFX1250-NEXT: v_pk_fma_bf16 v0, 0x42c83f80, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, <2 x bfloat> <bfloat 2.0, bfloat 200.0>)
ret <2 x bfloat> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
new file mode 100644
index 0000000000000..43692fb6e17f7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX1250-LABEL: fmin_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fmin_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmin_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fmin_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, s0, s1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vc(<2 x bfloat> %a) {
+; GFX1250-LABEL: fmin_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, 2.0 op_sel:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmin_v2bf16_vl(<2 x bfloat> %a) {
+; GFX1250-LABEL: fmin_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, 0x42c83f80, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX1250-LABEL: fmax_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fmax_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmax_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fmax_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, s0, s1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vc(<2 x bfloat> %a) {
+; GFX1250-LABEL: fmax_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, 2.0 op_sel:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>)
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
+; GFX1250-LABEL: fmax_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, 0x42c83f80, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)
+ ret <2 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 0cf8015d3b325..7ee7e7c3debd3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -161,22 +161,7 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b)
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX1250-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX1250-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_mul_f32 v1, v2, v3
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -237,24 +222,7 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inre
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_lshl_b32 s1, s0, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: v_dual_mul_f32 v0, s1, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_mul_f32_e32 v1, s0, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -327,30 +295,7 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_lshl_b32 s2, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s3, s1, 16
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_mul_f32 s2, s2, s3
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10010
-; GFX1250-NEXT: s_or_b32 s4, s2, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s3, s3, s2
-; GFX1250-NEXT: s_addk_co_i32 s3, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s2, s4, s3
-; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
-; GFX1250-NEXT: s_lshl_b32 s1, s1, 16
-; GFX1250-NEXT: s_lshr_b32 s2, s2, 16
-; GFX1250-NEXT: s_mul_f32 s0, s0, s1
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s3, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_pack_ll_b32_b16 s0, s2, s0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul <2 x bfloat> %a, %b
ret <2 x bfloat> %result
@@ -405,20 +350,7 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vc(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_mul_f32 v0, 0.5, v0 :: v_dual_mul_f32 v1, 0.5, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, 0.5 op_sel:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul <2 x bfloat> %a, <bfloat 0.5, bfloat 0.5>
ret <2 x bfloat> %result
@@ -473,20 +405,7 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vl(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-NEXT: v_dual_mul_f32 v0, 1.0, v0 :: v_dual_mul_f32 v1, 0x42c80000, v1
-; GFX1250-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX1250-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1250-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1250-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1250-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 0x42c83f80, v0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
ret <2 x bfloat> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
new file mode 100644
index 0000000000000..0dc93edb84cd6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+define amdgpu_ps bfloat @fneg_bf16_vv(bfloat %a) {
+; GFX9-LABEL: fneg_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg bfloat %a
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fneg_bf16_ss(bfloat inreg %a) {
+; GFX9-LABEL: fneg_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg bfloat %a
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fneg_v2bf16_vv(<2 x bfloat> %a) {
+; GFX9-LABEL: fneg_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg <2 x bfloat> %a
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fneg_v2bf16_ss(<2 x bfloat> inreg %a) {
+; GFX9-LABEL: fneg_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_xor_b32 s0, s0, 0x80008000
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_xor_b32 s0, s0, 0x80008000
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_xor_b32 s0, s0, 0x80008000
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg <2 x bfloat> %a
+ ret <2 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index dd8fa98c62afc..edfea5978833a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -1207,55 +1207,8 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<2 x bf16>) = COPY $vgpr1
- ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY]](<2 x bf16>)
- ; GFX1250-NEXT: [[UV2:%[0-9]+]]:_(bf16), [[UV3:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY1]](<2 x bf16>)
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV2]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[UV1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[UV3]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST9]]
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x bf16>)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x bf16>) = G_FADD [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0 = COPY [[FADD]](<2 x bf16>)
%0:_(<2 x bf16>) = COPY $vgpr0
%1:_(<2 x bf16>) = COPY $vgpr1
%2:_(<2 x bf16>) = G_FADD %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index 60e24cdc230d1..c6d7820a7dd28 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -856,46 +856,9 @@ body: |
; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
;
; GFX1250-LABEL: name: test_fcanonicalize_v2bf16
- ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST1]]
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST2]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST2]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(f32) = G_FCANONICALIZE [[BITCAST5]]
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FCANONICALIZE1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FCANONICALIZE1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST3]](bf16), [[BITCAST7]](bf16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ; GFX1250: [[DEF:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x bf16>) = G_FCANONICALIZE [[DEF]]
+ ; GFX1250-NEXT: S_NOP 0, implicit [[FCANONICALIZE]](<2 x bf16>)
%0:_(<2 x bf16>) = G_IMPLICIT_DEF
%1:_(<2 x bf16>) = G_FCANONICALIZE %0
S_NOP 0, implicit %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 5ae605b47d3cb..4b1697298d89f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1352,62 +1352,11 @@ body: |
; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
;
; GFX1250-LABEL: name: test_fma_v2bf16
- ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST10]](i16)
- ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
- ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
- ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
- ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST9]], [[BITCAST11]], [[BITCAST13]]
- ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST14]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST14]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST14]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST7]](bf16), [[BITCAST15]](bf16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ; GFX1250: [[DEF:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF2:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x bf16>) = G_FMA [[DEF]], [[DEF1]], [[DEF2]]
+ ; GFX1250-NEXT: S_NOP 0, implicit [[FMA]](<2 x bf16>)
%0:_(<2 x bf16>) = G_IMPLICIT_DEF
%1:_(<2 x bf16>) = G_IMPLICIT_DEF
%2:_(<2 x bf16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 57d941daf7718..b8b1787e4d8df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -1041,54 +1041,10 @@ body: |
; GFX11-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
;
; GFX1250-LABEL: name: test_fmul_v2bf16
- ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST7]], [[BITCAST9]]
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
- ; GFX1250-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x bf16>)
+ ; GFX1250: [[DEF:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x bf16>) = G_FMUL [[DEF]], [[DEF1]]
+ ; GFX1250-NEXT: S_NOP 0, implicit [[FMUL]](<2 x bf16>)
%0:_(<2 x bf16>) = G_IMPLICIT_DEF
%1:_(<2 x bf16>) = G_IMPLICIT_DEF
%2:_(<2 x bf16>) = G_FMUL %0, %1
>From 971e102f5f06eb8cc4199e3539f504ada4e27025 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Thu, 17 Sep 2026 13:59:44 -0500
Subject: [PATCH 2/5] Test fneg(fabs)
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll | 102 ++++++++++++++++++
1 file changed, 102 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
index 0dc93edb84cd6..31d064ea933bd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
@@ -100,3 +100,105 @@ define amdgpu_ps <2 x bfloat> @fneg_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = fneg <2 x bfloat> %a
ret <2 x bfloat> %result
}
+
+define amdgpu_ps bfloat @fneg_fabs_bf16_vv(bfloat %a) {
+; GFX9-LABEL: fneg_fabs_bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_or_b32_e32 v0, 0x8000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_fabs_bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_or_b16 v0.l, 0x8000, v0.l
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_fabs_bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_or_b16 v0.l, 0x8000, v0.l
+; GFX1250-NEXT: ; return to shader part epilog
+ %fabs = call bfloat @llvm.fabs.bf16(bfloat %a)
+ %result = fneg bfloat %fabs
+ ret bfloat %result
+}
+
+define amdgpu_ps bfloat @fneg_fabs_bf16_ss(bfloat inreg %a) {
+; GFX9-LABEL: fneg_fabs_bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_or_b32_e32 v0, 0x8000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_fabs_bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_bitset1_b32 s0, 15
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_fabs_bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_bitset1_b32 s0, 15
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %fabs = call bfloat @llvm.fabs.bf16(bfloat %a)
+ %result = fneg bfloat %fabs
+ ret bfloat %result
+}
+
+define amdgpu_ps <2 x bfloat> @fneg_fabs_v2bf16_vv(<2 x bfloat> %a) {
+; GFX9-LABEL: fneg_fabs_v2bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_fabs_v2bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_fabs_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
+ %result = fneg <2 x bfloat> %fabs
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fneg_fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
+; GFX9-LABEL: fneg_fabs_v2bf16_ss:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_or_b32 s0, s0, 0x80008000
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_fabs_v2bf16_ss:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_or_b32 s0, s0, 0x80008000
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_fabs_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_or_b32 s0, s0, 0x80008000
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
+ %result = fneg <2 x bfloat> %fabs
+ ret <2 x bfloat> %result
+}
>From 39fed984ae25cf67a1f68e2f388b3045b2e81e18 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Fri, 18 Sep 2026 18:26:23 -0500
Subject: [PATCH 3/5] Add v2bf16 testing for strict fops
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../AMDGPU/GlobalISel/strict_fadd.bf16.ll | 72 ++++++++++++++++++
.../AMDGPU/GlobalISel/strict_fma.bf16.ll | 73 +++++++++++++++++++
.../AMDGPU/GlobalISel/strict_fmul.bf16.ll | 72 ++++++++++++++++++
.../AMDGPU/GlobalISel/strict_fsub.bf16.ll | 33 +++++++++
4 files changed, 250 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
new file mode 100644
index 0000000000000..ae790fb363e33
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+; TODO: Add scalar bf16 tests when scalar bf16 support is added.
+; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+
+define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; GFX1250-LABEL: strict_fadd_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fadd_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> inreg %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fadd_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vc(<2 x bfloat> %a) #0 {
+; GFX1250-LABEL: strict_fadd_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 2.0 op_sel:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vl(<2 x bfloat> %a) #0 {
+; GFX1250-LABEL: strict_fadd_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 0x42c83f80
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
new file mode 100644
index 0000000000000..6eda1d592f4af
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+; TODO: Add scalar bf16 tests when scalar bf16 support is added.
+; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+
+define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vvv(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; GFX1250-LABEL: strict_fma_v2bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v1, v2
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vss(<2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) #0 {
+; GFX1250-LABEL: strict_fma_v2bf16_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, s0, s1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_sss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) #0 {
+; GFX1250-LABEL: strict_fma_v2bf16_sss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_pk_fma_bf16 v0, s0, s1, v0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vsc(<2 x bfloat> %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fma_v2bf16_vsc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, s0, 0.5 op_sel:[0,0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> <bfloat 0xR3F00, bfloat 0xR3F00>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vll(<2 x bfloat> %a) #0 {
+; GFX1250-LABEL: strict_fma_v2bf16_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, 1.0, 2.0 op_sel:[0,1,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0xR3F80, bfloat 0xR3F80>, <2 x bfloat> <bfloat 0xR4000, bfloat 0xR4000>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
new file mode 100644
index 0000000000000..0fc97f7731158
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+; TODO: Add scalar bf16 tests when scalar bf16 support is added.
+; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+
+define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; GFX1250-LABEL: strict_fmul_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fmul.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fmul_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fmul.v2bf16(<2 x bfloat> %a, <2 x bfloat> inreg %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fmul_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, s0, s1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fmul.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vc(<2 x bfloat> %a) #0 {
+; GFX1250-LABEL: strict_fmul_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, 0.5 op_sel:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fmul.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0.5, bfloat 0.5>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vl(<2 x bfloat> %a) #0 {
+; GFX1250-LABEL: strict_fmul_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, 0x42c83f80
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fmul.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
new file mode 100644
index 0000000000000..af3f46e7b2394
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+; TODO: Add scalar bf16 tests when scalar bf16 support is added.
+; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+
+define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; GFX1250-LABEL: strict_fsub_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fsub.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) #0 {
+; GFX1250-LABEL: strict_fsub_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <2 x bfloat> @llvm.experimental.constrained.fsub.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <2 x bfloat> %result
+}
+
+attributes #0 = { strictfp }
>From 21825de4880260b97b983f91a7d781a2cadc6638 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 21 Sep 2026 15:56:44 -0500
Subject: [PATCH 4/5] Clamp elements to 2 and test v4bf16
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 9 +-
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 102 ++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 114 ++++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 28 +++++
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 102 ++++++++++++++++
.../AMDGPU/GlobalISel/legalize-fadd.mir | 95 +--------------
.../AMDGPU/GlobalISel/legalize-fma.mir | 109 +----------------
.../AMDGPU/GlobalISel/legalize-fmul.mir | 93 +-------------
8 files changed, 364 insertions(+), 288 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 82576c4e2bd7b..c922d14576af8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -997,9 +997,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FPOpActions.legalFor({V2BF16});
- FCanonicalizeActions.legalFor({V2BF16});
- StrictFPOpActions.legalFor({V2BF16});
+ FPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ FCanonicalizeActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16,
+ 2);
+ StrictFPOpActions.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
@@ -1204,7 +1205,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16});
+ FSubActions.lowerFor({V2BF16}).clampMaxNumElements(0, BF16, 2);
}
if (ST.hasAnyPackedFP32Ops())
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 38fd0cecbf90a..04488ee933b39 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -410,3 +410,105 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
%result = fadd <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fadd_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_add_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_add_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v4bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <4 x bfloat> %a, %b
+ ret <4 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index 4401368f54421..c7dc27aa16340 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -454,3 +454,117 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
+
+define amdgpu_ps <4 x bfloat> @fma_v4bf16_vvv(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
+; GFX9-LABEL: fma_v4bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v12, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v12
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX9-NEXT: v_fma_f32 v2, v2, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v12
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v12
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX9-NEXT: v_fma_f32 v3, v3, v4, v5
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v12
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v4bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b16_e32 v9.l, v3.h
+; GFX12-NEXT: v_mov_b16_e32 v10.l, v5.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX12-NEXT: v_dual_fmac_f32 v8, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_bfe_u32 v6, v8, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v11, 0x400000, v8
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v8
+; GFX12-NEXT: v_add3_u32 v6, v6, v8, 0x7fff
+; GFX12-NEXT: v_fmac_f32_e32 v10, v1, v9
+; GFX12-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
+; GFX12-NEXT: v_fmac_f32_e32 v5, v7, v3
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX12-NEXT: v_bfe_u32 v6, v10, 16, 1
+; GFX12-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v10
+; GFX12-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: v_add3_u32 v6, v6, v10, 0x7fff
+; GFX12-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v10
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v4bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v3, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
+ ret <4 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index 43692fb6e17f7..e4bb63fa786b9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -130,3 +130,31 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
%result = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX1250-LABEL: fmin_v4bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
+ ret <4 x bfloat> %result
+}
+
+define amdgpu_ps <4 x bfloat> @fmax_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX1250-LABEL: fmax_v4bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
+ ret <4 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 7ee7e7c3debd3..9208c5c805460 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -410,3 +410,105 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vl(<2 x bfloat> %a) {
%result = fmul <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <4 x bfloat> @fmul_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX9-LABEL: fmul_v4bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v4bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_mul_f32_e32 v6, v6, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_mul_f32 v4, v4, v5
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v4bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_mul_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <4 x bfloat> %a, %b
+ ret <4 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index edfea5978833a..6702c344c979c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -1651,96 +1651,11 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr0_vgpr1
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(<4 x bf16>) = COPY $vgpr2_vgpr3
- ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(bf16), [[UV1:%[0-9]+]]:_(bf16), [[UV2:%[0-9]+]]:_(bf16), [[UV3:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
- ; GFX1250-NEXT: [[UV4:%[0-9]+]]:_(bf16), [[UV5:%[0-9]+]]:_(bf16), [[UV6:%[0-9]+]]:_(bf16), [[UV7:%[0-9]+]]:_(bf16) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(f32) = G_FADD [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FADD]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[UV1]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(f32) = G_FADD [[BITCAST7]], [[BITCAST9]]
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FADD1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[UV2]](bf16)
- ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
- ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
- ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](bf16)
- ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST14]](i16)
- ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
- ; GFX1250-NEXT: [[FADD2:%[0-9]+]]:_(f32) = G_FADD [[BITCAST13]], [[BITCAST15]]
- ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i32) = G_BITCAST [[FADD2]](f32)
- ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST16]], [[C1]]
- ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST16]], [[C]](i32)
- ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
- ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
- ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST16]], [[ADD4]]
- ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD2]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
- ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
- ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[UV3]](bf16)
- ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
- ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
- ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[UV7]](bf16)
- ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
- ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
- ; GFX1250-NEXT: [[FADD3:%[0-9]+]]:_(f32) = G_FADD [[BITCAST19]], [[BITCAST21]]
- ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FADD3]](f32)
- ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
- ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
- ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
- ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
- ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD6]]
- ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FADD3]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
- ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
- ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
- ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST17]](bf16), [[BITCAST23]](bf16)
- ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250-NEXT: [[UV:%[0-9]+]]:_(<2 x bf16>), [[UV1:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY]](<4 x bf16>)
+ ; GFX1250-NEXT: [[UV2:%[0-9]+]]:_(<2 x bf16>), [[UV3:%[0-9]+]]:_(<2 x bf16>) = G_UNMERGE_VALUES [[COPY1]](<4 x bf16>)
+ ; GFX1250-NEXT: [[FADD:%[0-9]+]]:_(<2 x bf16>) = G_FADD [[UV]], [[UV2]]
+ ; GFX1250-NEXT: [[FADD1:%[0-9]+]]:_(<2 x bf16>) = G_FADD [[UV1]], [[UV3]]
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[FADD]](<2 x bf16>), [[FADD1]](<2 x bf16>)
; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x bf16>)
%0:_(<4 x bf16>) = COPY $vgpr0_vgpr1
%1:_(<4 x bf16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 4b1697298d89f..ca1ba32ef7424 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1769,111 +1769,10 @@ body: |
; GFX11-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
;
; GFX1250-LABEL: name: test_fma_v4bf16
- ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST4]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[BITCAST1]], [[BITCAST3]], [[BITCAST5]]
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[FMA]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST6]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST6]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST10]](i16)
- ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
- ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
- ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
- ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(f32) = G_FMA [[BITCAST9]], [[BITCAST11]], [[BITCAST13]]
- ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i32) = G_BITCAST [[FMA1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST14]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST14]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST14]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST16]](i16)
- ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
- ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
- ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
- ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT8:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
- ; GFX1250-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT8]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL8]](i32)
- ; GFX1250-NEXT: [[FMA2:%[0-9]+]]:_(f32) = G_FMA [[BITCAST17]], [[BITCAST19]], [[BITCAST21]]
- ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FMA2]](f32)
- ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
- ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
- ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
- ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
- ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD4]]
- ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA2]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
- ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
- ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX1250-NEXT: [[BITCAST24:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT9:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST24]](i16)
- ; GFX1250-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT9]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST25:%[0-9]+]]:_(f32) = G_BITCAST [[SHL9]](i32)
- ; GFX1250-NEXT: [[BITCAST26:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT10:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST26]](i16)
- ; GFX1250-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT10]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST27:%[0-9]+]]:_(f32) = G_BITCAST [[SHL10]](i32)
- ; GFX1250-NEXT: [[BITCAST28:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT11:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST28]](i16)
- ; GFX1250-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT11]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST29:%[0-9]+]]:_(f32) = G_BITCAST [[SHL11]](i32)
- ; GFX1250-NEXT: [[FMA3:%[0-9]+]]:_(f32) = G_FMA [[BITCAST25]], [[BITCAST27]], [[BITCAST29]]
- ; GFX1250-NEXT: [[BITCAST30:%[0-9]+]]:_(i32) = G_BITCAST [[FMA3]](f32)
- ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST30]], [[C1]]
- ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST30]], [[C]](i32)
- ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
- ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
- ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST30]], [[ADD6]]
- ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMA3]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
- ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
- ; GFX1250-NEXT: [[BITCAST31:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST7]](bf16), [[BITCAST15]](bf16)
- ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST23]](bf16), [[BITCAST31]](bf16)
- ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250: [[DEF:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[FMA:%[0-9]+]]:_(<2 x bf16>) = G_FMA [[DEF]], [[DEF]], [[DEF]]
+ ; GFX1250-NEXT: [[FMA1:%[0-9]+]]:_(<2 x bf16>) = G_FMA [[DEF]], [[DEF]], [[DEF]]
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[FMA]](<2 x bf16>), [[FMA1]](<2 x bf16>)
; GFX1250-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
%0:_(<4 x bf16>) = G_IMPLICIT_DEF
%1:_(<4 x bf16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index b8b1787e4d8df..1f01ecfe5fb77 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -1344,95 +1344,10 @@ body: |
; GFX11-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
;
; GFX1250-LABEL: name: test_fmul_v4bf16
- ; GFX1250: [[DEF:%[0-9]+]]:_(bf16) = G_IMPLICIT_DEF
- ; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](i16)
- ; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX1250-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
- ; GFX1250-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST2]](i16)
- ; GFX1250-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT1]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST3:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
- ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST1]], [[BITCAST3]]
- ; GFX1250-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL]](f32)
- ; GFX1250-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4194304
- ; GFX1250-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST4]], [[C1]]
- ; GFX1250-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
- ; GFX1250-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; GFX1250-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
- ; GFX1250-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
- ; GFX1250-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[AND]], [[C3]]
- ; GFX1250-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[BITCAST4]], [[ADD]]
- ; GFX1250-NEXT: [[C4:%[0-9]+]]:_(f32) = G_FCONSTANT float 0.000000e+00
- ; GFX1250-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[FCMP]](i1), [[OR]], [[ADD1]]
- ; GFX1250-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[SELECT]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
- ; GFX1250-NEXT: [[BITCAST5:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC]](i16)
- ; GFX1250-NEXT: [[BITCAST6:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST6]](i16)
- ; GFX1250-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT2]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST7:%[0-9]+]]:_(f32) = G_BITCAST [[SHL2]](i32)
- ; GFX1250-NEXT: [[BITCAST8:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST8]](i16)
- ; GFX1250-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT3]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST9:%[0-9]+]]:_(f32) = G_BITCAST [[SHL3]](i32)
- ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST7]], [[BITCAST9]]
- ; GFX1250-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL1]](f32)
- ; GFX1250-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST10]], [[C1]]
- ; GFX1250-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C]](i32)
- ; GFX1250-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
- ; GFX1250-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[AND1]], [[C3]]
- ; GFX1250-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[BITCAST10]], [[ADD2]]
- ; GFX1250-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL1]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[FCMP1]](i1), [[OR1]], [[ADD3]]
- ; GFX1250-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[SELECT1]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
- ; GFX1250-NEXT: [[BITCAST11:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX1250-NEXT: [[BITCAST12:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST12]](i16)
- ; GFX1250-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT4]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST13:%[0-9]+]]:_(f32) = G_BITCAST [[SHL4]](i32)
- ; GFX1250-NEXT: [[BITCAST14:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST14]](i16)
- ; GFX1250-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT5]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST15:%[0-9]+]]:_(f32) = G_BITCAST [[SHL5]](i32)
- ; GFX1250-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST13]], [[BITCAST15]]
- ; GFX1250-NEXT: [[BITCAST16:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL2]](f32)
- ; GFX1250-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST16]], [[C1]]
- ; GFX1250-NEXT: [[LSHR4:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST16]], [[C]](i32)
- ; GFX1250-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
- ; GFX1250-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[AND2]], [[C3]]
- ; GFX1250-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[BITCAST16]], [[ADD4]]
- ; GFX1250-NEXT: [[FCMP2:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL2]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[FCMP2]](i1), [[OR2]], [[ADD5]]
- ; GFX1250-NEXT: [[LSHR5:%[0-9]+]]:_(i32) = G_LSHR [[SELECT2]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR5]](i32)
- ; GFX1250-NEXT: [[BITCAST17:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX1250-NEXT: [[BITCAST18:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST18]](i16)
- ; GFX1250-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT6]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST19:%[0-9]+]]:_(f32) = G_BITCAST [[SHL6]](i32)
- ; GFX1250-NEXT: [[BITCAST20:%[0-9]+]]:_(i16) = G_BITCAST [[DEF]](bf16)
- ; GFX1250-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST20]](i16)
- ; GFX1250-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ANYEXT7]], [[C]](i32)
- ; GFX1250-NEXT: [[BITCAST21:%[0-9]+]]:_(f32) = G_BITCAST [[SHL7]](i32)
- ; GFX1250-NEXT: [[FMUL3:%[0-9]+]]:_(f32) = G_FMUL [[BITCAST19]], [[BITCAST21]]
- ; GFX1250-NEXT: [[BITCAST22:%[0-9]+]]:_(i32) = G_BITCAST [[FMUL3]](f32)
- ; GFX1250-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST22]], [[C1]]
- ; GFX1250-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST22]], [[C]](i32)
- ; GFX1250-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
- ; GFX1250-NEXT: [[ADD6:%[0-9]+]]:_(i32) = G_ADD [[AND3]], [[C3]]
- ; GFX1250-NEXT: [[ADD7:%[0-9]+]]:_(i32) = G_ADD [[BITCAST22]], [[ADD6]]
- ; GFX1250-NEXT: [[FCMP3:%[0-9]+]]:_(i1) = G_FCMP floatpred(uno), [[FMUL3]](f32), [[C4]]
- ; GFX1250-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[FCMP3]](i1), [[OR3]], [[ADD7]]
- ; GFX1250-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[SELECT3]], [[C]](i32)
- ; GFX1250-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR7]](i32)
- ; GFX1250-NEXT: [[BITCAST23:%[0-9]+]]:_(bf16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST5]](bf16), [[BITCAST11]](bf16)
- ; GFX1250-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x bf16>) = G_BUILD_VECTOR [[BITCAST17]](bf16), [[BITCAST23]](bf16)
- ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x bf16>), [[BUILD_VECTOR1]](<2 x bf16>)
+ ; GFX1250: [[DEF:%[0-9]+]]:_(<2 x bf16>) = G_IMPLICIT_DEF
+ ; GFX1250-NEXT: [[FMUL:%[0-9]+]]:_(<2 x bf16>) = G_FMUL [[DEF]], [[DEF]]
+ ; GFX1250-NEXT: [[FMUL1:%[0-9]+]]:_(<2 x bf16>) = G_FMUL [[DEF]], [[DEF]]
+ ; GFX1250-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x bf16>) = G_CONCAT_VECTORS [[FMUL]](<2 x bf16>), [[FMUL1]](<2 x bf16>)
; GFX1250-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<4 x bf16>)
%0:_(<4 x bf16>) = G_IMPLICIT_DEF
%1:_(<4 x bf16>) = G_IMPLICIT_DEF
>From 8f5235c6b043b7e53e5c0fa0655948c47c58f93f Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 21 Sep 2026 15:57:13 -0500
Subject: [PATCH 5/5] Add fsub test
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll | 85 +++++++++++++++++++
1 file changed, 85 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
new file mode 100644
index 0000000000000..c120f166b17db
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefix=GFX1250 %s
+
+; TODO: Add scalar bf16 tests when scalar bf16 support is added.
+; TODO: Add GFX9 and GFX12 testing when scalar bf16 support is added.
+
+define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vv(<2 x bfloat> %a, <2 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v2bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vs(<2 x bfloat> %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fsub_v2bf16_vs:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, s0 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
+; GFX1250-LABEL: fsub_v2bf16_ss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <2 x bfloat> %a, %b
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
+; GFX1250-LABEL: fsub_v2bf16_vc:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 2.0 op_sel:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
+ ret <2 x bfloat> %result
+}
+
+define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
+; GFX1250-LABEL: fsub_v2bf16_vl:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, 0x42c83f80, v0 neg_lo:[1,0] neg_hi:[1,0]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
+ ret <2 x bfloat> %result
+}
+
+
+define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v4bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <4 x bfloat> %a, %b
+ ret <4 x bfloat> %result
+}
More information about the llvm-commits
mailing list