[llvm] [AMDGPU][GlobalISel] Legalize odd bf16 vectors (PR #226707)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 08:19:30 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: LU-JOHN
<details>
<summary>Changes</summary>
Simplify `moreElementsIf` and `clampMaxNumElements` to just `clampMaxNumElementsStrict` for `MinNumMaxNumIeee` and `MinNumMaxNum`.
Use `clampMaxNumElementsStrict` for `FSubActions`. This is the only functional change.
Add testing for v3bf16 and v5b16.
---
Patch is 49.80 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/226707.diff
14 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+3-9)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll (+25)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll (+200)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll (+72)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll (+224)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fminnum.bf16.ll (+51)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll (+81)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.bf16.ll (+26)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll (+156)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll (+28)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fadd.bf16.ll (+14)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.bf16.ll (+14)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fmul.bf16.ll (+14)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fsub.bf16.ll (+14)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c922d14576af8..3ae2515b18945 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1059,14 +1059,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// V2BF16
if (ST.hasBF16PackedInsts()) {
- MinNumMaxNumIeee.legalFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
- MinNumMaxNum.customFor({V2BF16})
- .moreElementsIf(all(elementTypeIs(0, BF16), isSmallOddVector(0)),
- oneMoreElement(0))
- .clampMaxNumElements(0, BF16, 2);
+ MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
+ MinNumMaxNum.customFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
MinNumMaxNumIeee.scalarize(0);
@@ -1205,7 +1199,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
if (ST.hasBF16PackedInsts()) {
- FSubActions.lowerFor({V2BF16}).clampMaxNumElements(0, BF16, 2);
+ FSubActions.lowerFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
}
if (ST.hasAnyPackedFP32Ops())
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
index 07f4e11691999..f7e40011531fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.bf16.ll
@@ -100,3 +100,28 @@ define amdgpu_ps <2 x bfloat> @fabs_v2bf16_ss(<2 x bfloat> inreg %a) {
%result = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %a)
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fabs_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fabs_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fabs_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX12-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fabs_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.fabs.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 04488ee933b39..9339b26ed66a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -411,6 +411,87 @@ define amdgpu_ps <2 x bfloat> @fadd_v2bf16_vl(<2 x bfloat> %a) {
ret <2 x bfloat> %result
}
+define amdgpu_ps <3 x bfloat> @fadd_v3bf16_vv(<3 x bfloat> %a, <3 x bfloat> %b) {
+; GFX9-LABEL: fadd_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v6, v6, v0, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v5, v5, v2, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v3
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <3 x bfloat> %a, %b
+ ret <3 x bfloat> %result
+}
define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-LABEL: fadd_v4bf16_vv:
; GFX9: ; %bb.0:
@@ -512,3 +593,122 @@ define amdgpu_ps <4 x bfloat> @fadd_v4bf16_vv(<4 x bfloat> %a, <4 x bfloat> %b)
%result = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %result
}
+define amdgpu_ps <5 x bfloat> @fadd_v5bf16_vv(<5 x bfloat> %a, <5 x bfloat> %b) {
+; GFX9-LABEL: fadd_v5bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX9-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX9-NEXT: v_add3_u32 v8, v8, v3, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v3
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v6, v6, v1, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX9-NEXT: v_add_f32_e32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_add3_u32 v7, v7, v4, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v4
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v7, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v11
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v3, v0, s0
+; GFX9-NEXT: v_perm_b32 v1, v4, v1, s0
+; GFX9-NEXT: v_alignbit_b32 v2, s0, v2, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fadd_v5bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX12-NEXT: v_dual_add_f32 v2, v2, v5 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX12-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-NEXT: v_dual_add_f32 v6, v6, v7 :: v_dual_lshlrev_b32 v7, 16, v1
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v6
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-NEXT: v_dual_cndmask_b32 v3, v9, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add_f32_e32 v1, v1, v4
+; GFX12-NEXT: v_add_f32_e32 v7, v7, v8
+; GFX12-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX12-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v7
+; GFX12-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX12-NEXT: v_add3_u32 v8, v9, v7, 0x7fff
+; GFX12-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v5, v10, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fadd_v5bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v3
+; GFX1250-NEXT: v_pk_add_bf16 v1, v1, v4
+; GFX1250-NEXT: v_pk_add_bf16 v2, v2, v5
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = fadd <5 x bfloat> %a, %b
+ ret <5 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 8e777f7b6e2a0..949999c21bdbf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -235,3 +235,75 @@ define amdgpu_ps <2 x bfloat> @fcanonicalize_v2bf16_s(<2 x bfloat> inreg %src) {
declare bfloat @llvm.canonicalize.bf16(bfloat)
declare <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat>)
+
+define amdgpu_ps <3 x bfloat> @fcanonicalize_v3bf16_vv(<3 x bfloat> %a) {
+; GFX9-LABEL: fcanonicalize_v3bf16_vv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v2, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_max_f32_e32 v1, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fcanonicalize_v3bf16_vv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v2, v2, v2
+; GFX12-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v1
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_dual_cndmask_b32 v1, v5, v6 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX12-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fcanonicalize_v3bf16_vv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: ; return to shader part epilog
+ %result = call <3 x bfloat> @llvm.canonicalize.v3bf16(<3 x bfloat> %a)
+ ret <3 x bfloat> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index c7dc27aa16340..3481f5fc096fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -455,6 +455,94 @@ define amdgpu_ps <2 x bfloat> @fma_v2bf16_vll(<2 x bfloat> %a) {
declare bfloat @llvm.fma.bf16(bfloat, bfloat, bfloat)
declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
+define amdgpu_ps <3 x bfloat> @fma_v3bf16_vvv(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) {
+; GFX9-LABEL: fma_v3bf16_vvv:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX9-NEXT: v_fma_f32 v2, v2, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
+; GFX9-NEXT: v_alignbit_b32 v1, s0, v1, 16
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX12-LABEL: fma_v3bf16_vvv:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX12-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX12-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX12-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX12-NEXT: v_fmac_f32_e32 v8, v6, v7
+; GFX12-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX12-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX12-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v8
+; GFX12-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
+; GFX12-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v5
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v0, v6, vcc_lo
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v4
+; GFX12-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
+; GFX12-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: fma_v3bf16_vvv:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v2, v4
+; GFX...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/226707
More information about the llvm-commits
mailing list