[llvm] [AMDGPU]{GlobalISel] Legalize odd bf16 vectors (PR #226707)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 08:18:39 PDT 2026
https://github.com/LU-JOHN created https://github.com/llvm/llvm-project/pull/226707
Simplify `moreElementsIf` and `clampMaxNumElements` to just `clampMaxNumElementsStrict` for `MinNumMaxNumIeee` and `MinNumMaxNum`.
Use `clampMaxNumElementsStrict` for `FSubActions`. This is the only functional change.
Add testing for v3bf16 and v5b16.
>From d1378cdf80b8a11eea525eaf500eaf1f277fefb0 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Sat, 26 Sep 2026 09:22:32 -0500
Subject: [PATCH] Use clampMaxNumElementsStrict
Signed-off-by: John Lu <John.Lu at amd.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 12 +-
.../CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll | 25 ++
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 200 ++++++++++++++++
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 72 ++++++
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 224 ++++++++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll | 51 ++++
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 81 +++++++
.../CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll | 26 ++
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 156 ++++++++++++
.../CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll | 28 +++
.../AMDGPU/GlobalISel/strict_fadd.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fma.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fmul.bf16.ll | 14 ++
.../AMDGPU/GlobalISel/strict_fsub.bf16.ll | 14 ++
14 files changed, 922 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c922d14576af8..3ae2515b18945 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1059,14 +1059,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// V2BF16
if (ST.hasBF16PackedInsts()) {
- MinNumMaxNumIeee.legalFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
- MinNumMaxNum.customFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
+ MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ MinNumMaxNum.customFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
MinNumMaxNumIeee.scalarize(0);
@@ -1205,7 +1199,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16}).clampMaxNumElements(0, BF16, 2);
+ FSubActions.lowerFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
if (ST.hasAnyPackedFP32Ops())
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
index 07f4e11691999..f7e40011531fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
@@ -100,3 +100,28 @@ define amdgpu_ps <2 x bfloat> @fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fabs_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fabs_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.fabs.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 04488ee933b39..9339b26ed66a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -411,6 +411,87 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fadd_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fadd_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-LABEL: fadd_v4bf16_vv:
; GFX9: ; %bb.0:
@@ -512,3 +593,122 @@ define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
%result = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %result
}
+define amdgpu_ps <5 x bfloat> @fadd_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX9-LABEL: fadd_v5bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v8, v8, v3, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX9-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v5bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-NEXT: v_dual_add_f32 v6, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-NEXT: v_dual_cndmask_b32 v3, v9, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX12-NEXT: v_add_f32_e32 v7, v7, v8
+; GFX12-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v7
+; GFX12-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: v_add3_u32 v8, v9, v7, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v10, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v5bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v3
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v4
+; GFX1250-NEXT: v_pk_add_bf16 v2, v2, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <5 x bfloat> %a, %b
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 8e777f7b6e2a0..949999c21bdbf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -235,3 +235,75 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
declare bfloat @llvm.canonicalize.bf16(bfloat)
declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
+
+define amdgpu_ps <3 x bfloat> @fcanonicalize_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fcanonicalize_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v2, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v2, v2, v2
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_dual_cndmask_b32 v1, v5, v6 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.canonicalize.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index c7dc27aa16340..3481f5fc096fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -455,6 +455,94 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
+define amdgpu_ps <3 x bfloat> @fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
+; GFX9-LABEL: fma_v3bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_fma_f32 v2, v2, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v3bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX12-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX12-NEXT: v_fmac_f32_e32 v8, v6, v7
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v8
+; GFX12-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v0, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v3bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v3, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c)
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fma_v4bf16_vvv(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) {
; GFX9-LABEL: fma_v4bf16_vvv:
; GFX9: ; %bb.0:
@@ -568,3 +656,139 @@ define amdgpu_ps <4 x bfloat> @fma_v4bf16_vvv(<4 x bfloat> %a, <4 x bfloat> %b,
%result = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %result
}
+
+define amdgpu_ps <5 x bfloat> @fma_v5bf16_vvv(<5 x bfloat> %a, <5 x bfloat> %b, <5 x bfloat> %c) {
+; GFX9-LABEL: fma_v5bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_fma_f32 v0, v0, v3, v6
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v15, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX9-NEXT: v_fma_f32 v3, v3, v6, v9
+; GFX9-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v9, v9, v3, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX9-NEXT: v_fma_f32 v1, v1, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v7
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v12
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
+; GFX9-NEXT: v_fma_f32 v4, v4, v6, v7
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_fma_f32 v2, v2, v5, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v15
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v5bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX12-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_fmac_f32_e32 v6, v0, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_dual_fmac_f32 v12, v0, v3 :: v_dual_lshlrev_b32 v3, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v7.h
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX12-NEXT: v_fmac_f32_e32 v5, v2, v4
+; GFX12-NEXT: v_fmac_f32_e32 v11, v9, v10
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v12
+; GFX12-NEXT: v_bfe_u32 v9, v11, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v11
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v11
+; GFX12-NEXT: v_fmac_f32_e32 v1, v0, v3
+; GFX12-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX12-NEXT: v_add3_u32 v9, v9, v11, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v9, v9, v10, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v8, v10, v6, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v10, v12, 16, 1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
+; GFX12-NEXT: v_add3_u32 v6, v10, v12, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v12
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v9.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v5bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v3, v6
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v4, v7
+; GFX1250-NEXT: v_pk_fma_bf16 v2, v2, v5, v8
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <5 x bfloat> @llvm.fma.v5bf16(<5 x bfloat> %a, <5 x bfloat> %b, <5 x bfloat> %c)
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
index e4bb63fa786b9..f87116aad81ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll
@@ -131,6 +131,57 @@ define amdgpu_ps <2 x bfloat> @fmax_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fmin_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmin_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX9-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmin_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX12-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmin_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_min_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
+ ret <3 x bfloat> %result
+}
+
+define amdgpu_ps <3 x bfloat> @fmax_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmax_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX9-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmax_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmax_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fmin_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX1250-LABEL: fmin_v4bf16_vv:
; GFX1250: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 9208c5c805460..1cd14c61e7f24 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -411,6 +411,87 @@ define amdgpu_ps <2 x bfloat> @fmul_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fmul_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fmul_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fmul_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fmul_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_mul_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fmul <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fmul_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-LABEL: fmul_v4bf16_vv:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
index 31d064ea933bd..d1fbfce1c8fc6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll
@@ -202,3 +202,29 @@ define amdgpu_ps <2 x bfloat> @fneg_fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = fneg <2 x bfloat> %fabs
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <3 x bfloat> @fneg_v3bf16_v(<3 x bfloat> %a) {
+; GFX9-LABEL: fneg_v3bf16_v:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fneg_v3bf16_v:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX12-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fneg_v3bf16_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX1250-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fneg <3 x bfloat> %a
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index 06c2495ab4bff..ad20a1ec9e330 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -684,3 +684,159 @@ define amdgpu_ps <2 x bfloat> @fptrunc_v2f32_to_v2bf16_v(<2 x float> %a) {
%result = fptrunc <2 x float> %a to <2 x bfloat>
ret <2 x bfloat> %result
}
+
+define amdgpu_ps <3 x bfloat> @fptrunc_v3f32_to_v3bf16(<3 x float> %a) {
+; GFX9-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX9-NEXT: v_cvt_pk_bf16_f32 v1, v2, v2
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX1250-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX1250-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX1250-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX1250-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
+; GFX1250-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX1250-FAKE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_alignbit_b32 v1, s0, v2, 16
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_v3f32_to_v3bf16:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX1250-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX1250-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX1250-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX1250-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX1250-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
+ %result = fptrunc <3 x float> %a to <3 x bfloat>
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
index c120f166b17db..a9fba2761f7fc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -70,6 +70,19 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
}
+define amdgpu_ps <3 x bfloat> @fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX1250-LABEL: fsub_v4bf16_vv:
; GFX1250: ; %bb.0:
@@ -83,3 +96,18 @@ define amdgpu_ps <4 x bfloat> @fsub_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
%result = fsub <4 x bfloat> %a, %b
ret <4 x bfloat> %result
}
+
+define amdgpu_ps <5 x bfloat> @fsub_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX1250-LABEL: fsub_v5bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v2, v2, v5 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fsub <5 x bfloat> %a, %b
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
index ae790fb363e33..603166cc8dc17 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll
@@ -70,3 +70,17 @@ define amdgpu_ps <2 x bfloat> @strict_fadd_v2bf16_vl(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fadd_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fadd_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fadd.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
index 6eda1d592f4af..f8e9cd21ff7ee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll
@@ -71,3 +71,17 @@ define amdgpu_ps <2 x bfloat> @strict_fma_v2bf16_vll(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
+; GFX1250-LABEL: strict_fma_v3bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v3, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
index 0fc97f7731158..9cc2fb662c2db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll
@@ -70,3 +70,17 @@ define amdgpu_ps <2 x bfloat> @strict_fmul_v2bf16_vl(<2 x bfloat> %a) #0 {
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fmul_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fmul_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_mul_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fmul.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
index af3f46e7b2394..6a75044c7a325 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll
@@ -31,3 +31,17 @@ define amdgpu_ps <2 x bfloat> @strict_fsub_v2bf16_ss(<2 x bfloat> inreg %a, <2 x
}
attributes #0 = { strictfp }
+
+define amdgpu_ps <3 x bfloat> @strict_fsub_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX1250-LABEL: strict_fsub_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.experimental.constrained.fsub.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
+ ret <3 x bfloat> %result
+}
More information about the llvm-commits
mailing list