[llvm] 9afffc9 - [AMDGPU][GISel] RegBankLegalize rules for BF16 variants of transcendental intrinsics (#214040)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 09:50:23 PDT 2026
Author: Chinmay Deshpande
Date: 2026-08-05T09:50:17-07:00
New Revision: 9afffc9ca0a9469fe215572e089d819f40663e0e
URL: https://github.com/llvm/llvm-project/commit/9afffc9ca0a9469fe215572e089d819f40663e0e
DIFF: https://github.com/llvm/llvm-project/commit/9afffc9ca0a9469fe215572e089d819f40663e0e.diff
LOG: [AMDGPU][GISel] RegBankLegalize rules for BF16 variants of transcendental intrinsics (#214040)
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 0b1a9e4904078..499a018828957 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -110,6 +110,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniformAtDef(Reg);
case UniS128:
return MRI.getType(Reg) == LLT::scalar(128) && MUI.isUniformAtDef(Reg);
+ case UniBF16:
+ return MRI.getType(Reg).isBFloat16() && MUI.isUniformAtDef(Reg);
case UniP0:
return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniformAtDef(Reg);
case UniP1:
@@ -1988,15 +1990,16 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
- addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp}, Standard)
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}})
- .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}});
+ addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS64}, {{UniInVgprS64}, {IntrId, Vgpr64}}})
+ .Any({{DivS64}, {{Vgpr64}, {IntrId, Vgpr64}}});
addRulesForIOpcs({amdgcn_mul_u24, amdgcn_mul_i24}, Standard)
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
@@ -2511,31 +2514,34 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Div(S64, {{Vgpr64}, {IntrId, Vgpr64, Vgpr32}})
.Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32}});
- addRulesForIOpcs({amdgcn_exp2}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST);
-
- addRulesForIOpcs({amdgcn_rcp, amdgcn_sqrt}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST)
- .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}})
- .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}});
-
- addRulesForIOpcs({amdgcn_log}, Standard)
- .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
- .Uni(S16, {{Sgpr16}, {IntrId, Sgpr16}}, hasPST)
- .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}}, !hasPST)
- .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
- .Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}}, hasPST)
- .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}}, !hasPST);
+ addRulesForIOpcs({amdgcn_exp2})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST);
+
+ addRulesForIOpcs({amdgcn_rcp, amdgcn_sqrt})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST)
+ .Any({{DivS64}, {{Vgpr64}, {IntrId, Vgpr64}}})
+ .Any({{UniS64}, {{UniInVgprS64}, {IntrId, Vgpr64}}});
+
+ addRulesForIOpcs({amdgcn_log})
+ .Any({{DivS16}, {{Vgpr16}, {IntrId, Vgpr16}}})
+ .Any({{UniBF16}, {{UniInVgprS16}, {IntrId, Vgpr16}}})
+ .Any({{UniS16}, {{Sgpr16}, {IntrId, Sgpr16}}}, hasPST)
+ .Any({{UniS16}, {{UniInVgprS16}, {IntrId, Vgpr16}}}, !hasPST)
+ .Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}})
+ .Any({{UniS32}, {{Sgpr32}, {IntrId, Sgpr32}}}, hasPST)
+ .Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}}, !hasPST);
addRulesForIOpcs({amdgcn_ds_atomic_async_barrier_arrive_b64})
.Any({{}, {{}, {IntrId, VgprP3}}});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 16a5634a8eb62..93c3690d659a0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -52,6 +52,8 @@ enum UniformityLLTOpPredicateID {
UniS64,
UniS128,
+ UniBF16,
+
DivS1,
DivS16,
DivS32,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 9f2d3af93b02d..a9ee4dbe48316 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4897,9 +4897,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case Intrinsic::amdgcn_rcp:
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_sqrt: {
- unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
- if (Subtarget.hasPseudoScalarTrans() && (Size == 16 || Size == 32) &&
- isSALUMapping(MI))
+ LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+ unsigned Size = Ty.getSizeInBits();
+ // There is no pseudo scalar transcendental instruction for bf16.
+ if (Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
+ (Size == 16 || Size == 32) && isSALUMapping(MI))
return getDefaultMappingSOP(MI);
return getDefaultMappingVOP(MI);
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
index 715a1e94d5c78..693e60c004428 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.exp2.bf16(bfloat) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
index db198df25c5d7..809987f2f42da 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.log.bf16(bfloat) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
index 100ec593c311f..a6ab89b9b864b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=GI-TRUE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=GI-FAKE16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefix=GI-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefix=GI-FAKE16 %s
declare bfloat @llvm.amdgcn.rcp.bf16(bfloat) #0
@@ -32,6 +30,98 @@ define amdgpu_kernel void @rcp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v0, s2
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, s2
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, s2
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
+ %rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %src) #0
+ store bfloat %rcp, ptr addrspace(1) %out, align 2
+ ret void
+}
+
+define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrspace(1) %in) #1 {
+; SDAG-TRUE16-LABEL: rcp_bf16_global_load:
+; SDAG-TRUE16: ; %bb.0:
+; SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; SDAG-TRUE16-NEXT: v_nop
+; SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, v0.l
+; SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; SDAG-TRUE16-NEXT: s_endpgm
+;
+; SDAG-FAKE16-LABEL: rcp_bf16_global_load:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v0, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16_global_load:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; GI-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, v0.l
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16_global_load:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; GI-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, v0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %src.ptr = getelementptr bfloat, ptr addrspace(1) %in, i32 %tid
+ %src = load bfloat, ptr addrspace(1) %src.ptr, align 2
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %src) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -60,6 +150,30 @@ define amdgpu_kernel void @rcp_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16_constant_4:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, 4.0
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16_constant_4:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, 4.0
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat 4.0) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -88,6 +202,30 @@ define amdgpu_kernel void @rcp_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_bf16_constant_100:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, 0x42c8
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_bf16_constant_100:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, 0x42c8
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat 100.0) #0
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
@@ -116,6 +254,30 @@ define amdgpu_kernel void @rcp_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GI-TRUE16-LABEL: rcp_undef_bf16:
+; GI-TRUE16: ; %bb.0:
+; GI-TRUE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-TRUE16-NEXT: v_nop
+; GI-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, s0
+; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: s_endpgm
+;
+; GI-FAKE16-LABEL: rcp_undef_bf16:
+; GI-FAKE16: ; %bb.0:
+; GI-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GI-FAKE16-NEXT: v_nop
+; GI-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GI-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GI-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, s0
+; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: s_endpgm
%rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat undef)
store bfloat %rcp, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
index ff44369210826..e6b76785aa116 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=SDAG-FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s
declare bfloat @llvm.amdgcn.rsq.bf16(bfloat) #0
@@ -32,6 +30,30 @@ define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, s2
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, s2
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, s2
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %src) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -61,6 +83,30 @@ define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16_constant_4:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 4.0
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16_constant_4:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, 4.0
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 4.0) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -90,6 +136,30 @@ define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_bf16_constant_100:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 0x42c8
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_bf16_constant_100:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, 0x42c8
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 100.0) #0
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
@@ -109,6 +179,30 @@ define amdgpu_kernel void @rsq_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_endpgm
+;
+; GISEL-REAL16-LABEL: rsq_undef_bf16:
+; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-REAL16-NEXT: v_nop
+; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
+; GISEL-REAL16-NEXT: v_rsq_bf16_e32 v0.l, s0
+; GISEL-REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT: s_endpgm
+;
+; GISEL-FAKE16-LABEL: rsq_undef_bf16:
+; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GISEL-FAKE16-NEXT: v_nop
+; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-FAKE16-NEXT: v_rsq_bf16_e32 v0, s0
+; GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat undef)
store bfloat %rsq, ptr addrspace(1) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
index 50af8bd63651b..7f5568a31c138 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
@@ -1,10 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
-
-; FIXME: GlobalISel does not work with bf16
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
declare bfloat @llvm.amdgcn.sqrt.bf16(bfloat) #0
More information about the llvm-commits
mailing list