[llvm] [AMDGPU][GISel] RegBankLegalize rules for BF16 variants of transcendental intrinsics (PR #214040)
Chinmay Deshpande via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 12:02:28 PDT 2026
https://github.com/chinmaydd created https://github.com/llvm/llvm-project/pull/214040
None
>From cd26cfb1012d5b092609b6878d759589d91c77ea Mon Sep 17 00:00:00 2001
From: Chinmay Deshpande <chdeshpa at amd.com>
Date: Tue, 4 Aug 2026 11:57:45 -0700
Subject: [PATCH] [AMDGPU][GISel] RegBankLegalize rules for BF16 variants of
transcendental functions
---
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 74 +++++++------
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 5 +
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll | 102 +++++++++++++++++-
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll | 102 +++++++++++++++++-
.../CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll | 6 +-
8 files changed, 252 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 881afaf920037..cfa3004b4da04 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -110,6 +110,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniformAtDef(Reg);
case UniS128:
return MRI.getType(Reg) == LLT::scalar(128) && MUI.isUniformAtDef(Reg);
+ case UniBF16:
+ return MRI.getType(Reg).isBFloat16() && MUI.isUniformAtDef(Reg);
case UniP0:
return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniformAtDef(Reg);
case UniP1:
@@ -1988,15 +1990,16 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
- addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp}, Standard)
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}})
- .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}});
+ addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS64}, {{UniInVgprS64}, {IntrId, Vgpr64}}})
+ .Any({{DivS64}, {{Vgpr64}, {IntrId, Vgpr64}}});
addRulesForIOpcs({amdgcn_mul_u24, amdgcn_mul_i24}, Standard)
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
@@ -2476,31 +2479,34 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(S64, {{Vgpr64}, {IntrId, Vgpr64, Vgpr32}})
.Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32}});
- addRulesForIOpcs({amdgcn_exp2}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST);
-
- addRulesForIOpcs({amdgcn_rcp, amdgcn_sqrt}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST)
- .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}})
- .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}});
-
- addRulesForIOpcs({amdgcn_log}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST);
+ addRulesForIOpcs({amdgcn_exp2})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST);
+
+ addRulesForIOpcs({amdgcn_rcp, amdgcn_sqrt})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST)
+ .Any({{DivS64}, {{Vgpr64}, {IntrId, Vgpr64}}})
+ .Any({{UniS64}, {{UniInVgprS64}, {IntrId, Vgpr64}}});
+
+ addRulesForIOpcs({amdgcn_log})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST);
addRulesForIOpcs({amdgcn_ds_atomic_async_barrier_arrive_b64})
.Any({{}, {{}, {IntrId, VgprP3}}});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 16a5634a8eb62..88e90b610211b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -51,6 +51,11 @@ enum UniformityLLTOpPredicateID {
UniS32,
UniS64,
UniS128,
+ // Matches bfloat16 only, unlike UniS16 which matches any uniform 16-bit
+ // scalar. Used by the transcendentals, which have no pseudo scalar bfloat16
+ // form and so have to stay on the VALU. Order rules using this ahead of the
+ // UniS16 rule they carve out of, and note that fast rules cannot express it.
+ UniBF16,
DivS1,
DivS16,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 9f2d3af93b02d..a9ee4dbe48316 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4897,9 +4897,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case Intrinsic::amdgcn_rcp:
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_sqrt: {
- unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
- if (Subtarget.hasPseudoScalarTrans() && (Size == 16 || Size == 32) &&
- isSALUMapping(MI))
+ LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+ unsigned Size = Ty.getSizeInBits();
+ // There is no pseudo scalar transcendental instruction for bf16.
+ if (Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
+ (Size == 16 || Size == 32) && isSALUMapping(MI))
return getDefaultMappingSOP(MI);
return getDefaultMappingVOP(MI);
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
index 173d8cd72f801..879116778970a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.exp2.bf16(bfloat) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
index fda22156ab62f..fbb0002aac024 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.log.bf16(bfloat) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
index 4dbc7de5ca7df..45f4a3be070ea 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=GI-TRUE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=GI-FAKE16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=GI-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=GI-FAKE16 %s
declare bfloat @llvm.amdgcn.rcp.bf16(bfloat) #0
@@ -32,6 +30,30 @@ define amdgpu_kernel void @rcp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v0, s2
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, s2
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, s2
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %src) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -60,6 +82,30 @@ define amdgpu_kernel void @rcp_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16_constant_4:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, 4.0
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16_constant_4:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, 4.0
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat 4.0) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -88,6 +134,30 @@ define amdgpu_kernel void @rcp_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16_constant_100:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, 0x42c8
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16_constant_100:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, 0x42c8
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat 100.0) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -116,6 +186,30 @@ define amdgpu_kernel void @rcp_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_undef_bf16:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, s0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_undef_bf16:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, s0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat undef)
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
index f4b6fde318b47..2c77c2e98cf67 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=SDAG-FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s
declare bfloat @llvm.amdgcn.rsq.bf16(bfloat) #0
@@ -32,6 +30,30 @@ define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, s2
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, s2
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, s2
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %src) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -61,6 +83,30 @@ define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16_constant_4:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 4.0
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16_constant_4:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, 4.0
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 4.0) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -90,6 +136,30 @@ define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16_constant_100:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 0x42c8
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16_constant_100:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, 0x42c8
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 100.0) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -109,6 +179,30 @@ define amdgpu_kernel void @rsq_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_undef_bf16:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, s0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_undef_bf16:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, s0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat undef)
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
index acf36cc076f56..908943a4bfe7c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.sqrt.bf16(bfloat) #0
More information about the llvm-commits
mailing list