[llvm] [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on gfx1250/gfx13 (PR #209128)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 03:04:12 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Harrison Hao (harrisonGPU)
<details>
<summary>Changes</summary>
Targets with packed bf16 instructions can do a scalar bf16 fadd with a single `v_pk_add_bf16` operating on the low half, instead of promoting to f32 and going through `v_fma_mix_f32_bf16 + v_cvt_pk_bf16_f32`.
---
Patch is 40.67 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209128.diff
8 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+9)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+13)
- (modified) llvm/lib/Target/AMDGPU/SIInstructions.td (+11-1)
- (modified) llvm/lib/Target/AMDGPU/VOP3PInstructions.td (+4-2)
- (modified) llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll (+28-36)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+56-34)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll (+39-68)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll (+49)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4b2fd87c5b078..7129a3738eab8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -236,6 +236,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Use v_pk_add_bf16 for scalar fadd.
+ setOperationAction(ISD::FADD, MVT::bf16, Legal);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 81b7bf7f769ad..da7a8358b967e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2416,6 +2416,19 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
break;
}
+ case AMDGPU::V_ADD_BF16_PSEUDO_e64: {
+ assert(ST.hasBF16PackedInsts());
+ MI.setDesc(get(AMDGPU::V_PK_ADD_BF16));
+ MI.addOperand(MachineOperand::CreateImm(0)); // op_sel
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi
+ MachineOperand *Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
+ MachineOperand *Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
+ break;
+ }
+
case AMDGPU::GET_STACK_BASE:
// The stack starts at offset 0 unless we need to reserve some space at the
// bottom.
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d3ea76ebb647d..e252f4b8a26cf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2113,7 +2113,7 @@ def : ClampPat<V_MAX_F16_fake16_e64, f16>;
let True16Predicate = UseFakeTrue16Insts in
def : ClampPat<V_MAX_BF16_PSEUDO_e64, bf16>;
let True16Predicate = UseRealTrue16Insts,
- SubtargetPredicate = HasBF16PackedInsts in
+ SubtargetPredicate = HasBF16PackedInsts in {
def : GCNPat<(bf16 (AMDGPUclamp (VOP3Mods bf16:$src0, i32:$src0_modifiers))),
(EXTRACT_SUBREG
(V_PK_MAX_NUM_BF16
@@ -2121,6 +2121,16 @@ let True16Predicate = UseRealTrue16Insts,
$src0_modifiers, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
DSTCLAMP.ENABLE), lo16)
>;
+ def : GCNPat<
+ (bf16 (fadd (bf16 (VOP3Mods bf16:$src0, i32:$src0_mods)),
+ (bf16 (VOP3Mods bf16:$src1, i32:$src1_mods)))),
+ (EXTRACT_SUBREG
+ (V_PK_ADD_BF16
+ $src0_mods, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ $src1_mods, (REG_SEQUENCE VGPR_32, $src1, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ DSTCLAMP.NONE), lo16)
+ >;
+}
let SubtargetPredicate = HasVOP3PInsts in {
def : GCNPat <
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index f99367d198f26..b64d9f3e13fac 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1458,8 +1458,10 @@ let isCommutable = 1, isReMaterializable = 1 in {
// Scalar pseudo used to emulate AMDGPUClamp.
// Expanded to V_PK_MAX_NUM_BF16 with unused high half.
// FIXME-TRUE16: Pseudo expansion of this won't work with True16.
- let True16Predicate = UseFakeTrue16Insts in
- defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ let True16Predicate = UseFakeTrue16Insts in {
+ defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ defm V_ADD_BF16_PSEUDO : VOP3Inst <"v_add_bf16", VOP_BF16_BF16_BF16, any_fadd>;
+ }
}
let SubtargetPredicate = HasPackedFP64Ops, SchedRW = [WriteDouble] in {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index cb6e860c92cc1..9feeb3ce0dbcc 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15629,33 +15629,31 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16: ; %bb.0:
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1364-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1364-TRUE16-NEXT: s_not_b32 s9, s2
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, s3
; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1364-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15667,7 +15665,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1364-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX1364-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX1364-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1364-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -15695,15 +15693,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -15725,43 +15721,41 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-TRUE16: ; %bb.0:
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1332-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1332-TRUE16-NEXT: s_not_b32 s3, s3
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1332-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1332-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1332-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1332-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -15790,14 +15784,12 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-FAKE16-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index dc125ec5840ae..87ddda1ed5f40 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9254,14 +9254,19 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fadd bfloat %a, %b
ret bfloat %op
}
@@ -13400,15 +13405,21 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 1.0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x3f80
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, 1.0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
}
@@ -13511,15 +13522,21 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 0x42280000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4228
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, 0x4228, v0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 42.0
ret bfloat %add
}
@@ -13630,14 +13647,19 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fsub_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v1, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fsub_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fsub_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fsub bfloat %a, %b
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index e385261b9bf8a..1d95592c8ae53 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13693,12 +13693,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -13738,12 +13737,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(V...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/209128
More information about the llvm-commits
mailing list