[llvm] [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on gfx1250/gfx13 (PR #209128)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 03:03:37 PDT 2026
https://github.com/harrisonGPU created https://github.com/llvm/llvm-project/pull/209128
Targets with packed bf16 instructions can do a scalar bf16 fadd with a single `v_pk_add_bf16` operating on the low half, instead of promoting to f32 and going through `v_fma_mix_f32_bf16 + v_cvt_pk_bf16_f32`.
>From b563449fa89f079f82b452ee9b55bb61eb27172d Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 13 Jul 2026 17:59:28 +0800
Subject: [PATCH] [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on
gfx1250/gfx13
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 9 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++
llvm/lib/Target/AMDGPU/SIInstructions.td | 12 +-
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 6 +-
.../atomic_optimizations_global_pointer.ll | 64 +++++------
llvm/test/CodeGen/AMDGPU/bf16.ll | 90 +++++++++------
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 107 +++++++-----------
.../CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll | 49 ++++++++
8 files changed, 209 insertions(+), 141 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4b2fd87c5b078..7129a3738eab8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -236,6 +236,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Use v_pk_add_bf16 for scalar fadd.
+ setOperationAction(ISD::FADD, MVT::bf16, Legal);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 81b7bf7f769ad..da7a8358b967e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2416,6 +2416,19 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
break;
}
+ case AMDGPU::V_ADD_BF16_PSEUDO_e64: {
+ assert(ST.hasBF16PackedInsts());
+ MI.setDesc(get(AMDGPU::V_PK_ADD_BF16));
+ MI.addOperand(MachineOperand::CreateImm(0)); // op_sel
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi
+ MachineOperand *Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
+ MachineOperand *Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
+ break;
+ }
+
case AMDGPU::GET_STACK_BASE:
// The stack starts at offset 0 unless we need to reserve some space at the
// bottom.
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d3ea76ebb647d..e252f4b8a26cf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2113,7 +2113,7 @@ def : ClampPat<V_MAX_F16_fake16_e64, f16>;
let True16Predicate = UseFakeTrue16Insts in
def : ClampPat<V_MAX_BF16_PSEUDO_e64, bf16>;
let True16Predicate = UseRealTrue16Insts,
- SubtargetPredicate = HasBF16PackedInsts in
+ SubtargetPredicate = HasBF16PackedInsts in {
def : GCNPat<(bf16 (AMDGPUclamp (VOP3Mods bf16:$src0, i32:$src0_modifiers))),
(EXTRACT_SUBREG
(V_PK_MAX_NUM_BF16
@@ -2121,6 +2121,16 @@ let True16Predicate = UseRealTrue16Insts,
$src0_modifiers, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
DSTCLAMP.ENABLE), lo16)
>;
+ def : GCNPat<
+ (bf16 (fadd (bf16 (VOP3Mods bf16:$src0, i32:$src0_mods)),
+ (bf16 (VOP3Mods bf16:$src1, i32:$src1_mods)))),
+ (EXTRACT_SUBREG
+ (V_PK_ADD_BF16
+ $src0_mods, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ $src1_mods, (REG_SEQUENCE VGPR_32, $src1, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ DSTCLAMP.NONE), lo16)
+ >;
+}
let SubtargetPredicate = HasVOP3PInsts in {
def : GCNPat <
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index f99367d198f26..b64d9f3e13fac 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1458,8 +1458,10 @@ let isCommutable = 1, isReMaterializable = 1 in {
// Scalar pseudo used to emulate AMDGPUClamp.
// Expanded to V_PK_MAX_NUM_BF16 with unused high half.
// FIXME-TRUE16: Pseudo expansion of this won't work with True16.
- let True16Predicate = UseFakeTrue16Insts in
- defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ let True16Predicate = UseFakeTrue16Insts in {
+ defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ defm V_ADD_BF16_PSEUDO : VOP3Inst <"v_add_bf16", VOP_BF16_BF16_BF16, any_fadd>;
+ }
}
let SubtargetPredicate = HasPackedFP64Ops, SchedRW = [WriteDouble] in {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index cb6e860c92cc1..9feeb3ce0dbcc 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15629,33 +15629,31 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16: ; %bb.0:
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1364-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1364-TRUE16-NEXT: s_not_b32 s9, s2
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, s3
; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1364-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15667,7 +15665,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1364-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX1364-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX1364-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1364-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -15695,15 +15693,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -15725,43 +15721,41 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-TRUE16: ; %bb.0:
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1332-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1332-TRUE16-NEXT: s_not_b32 s3, s3
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1332-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1332-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1332-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1332-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -15790,14 +15784,12 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-FAKE16-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index dc125ec5840ae..87ddda1ed5f40 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9254,14 +9254,19 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fadd bfloat %a, %b
ret bfloat %op
}
@@ -13400,15 +13405,21 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 1.0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x3f80
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, 1.0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
}
@@ -13511,15 +13522,21 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 0x42280000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4228
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, 0x4228, v0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 42.0
ret bfloat %add
}
@@ -13630,14 +13647,19 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fsub_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v1, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fsub_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fsub_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fsub bfloat %a, %b
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index e385261b9bf8a..1d95592c8ae53 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13693,12 +13693,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -13738,12 +13737,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14280,12 +14278,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14325,12 +14322,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14888,12 +14884,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14934,12 +14929,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -15496,12 +15490,10 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v4, v4, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -15540,12 +15532,10 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16068,12 +16058,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16112,12 +16100,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16661,12 +16647,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16706,12 +16690,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17246,10 +17228,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v3, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17280,10 +17261,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17726,10 +17706,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v3, 1.0, v4 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17758,11 +17737,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -18200,12 +18177,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18245,12 +18221,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18813,12 +18788,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18857,12 +18830,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 03897e4ee9a24..08daa1a23778c 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -531,3 +531,52 @@ define <4 x bfloat> @v_maxnum_v4bf16_neg1(<4 x bfloat> %a, <4 x bfloat> %b) {
%max = call <4 x bfloat> @llvm.maxnum.v4bf16( <4 x bfloat> %a, <4 x bfloat> %neg)
ret <4 x bfloat> %max
}
+
+define bfloat @v_fadd_bf16_neg(bfloat %a, bfloat %b) {
+; GFX1250-LABEL: v_fadd_bf16_neg:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_neg:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %neg = fneg bfloat %b
+ %res = fadd bfloat %a, %neg
+ ret bfloat %res
+}
+
+define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
+; GFX1250-LABEL: v_fadd_bf16_chain:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_chain:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1310-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %num = fadd bfloat %a, %b
+ %res = fadd bfloat %num, %c
+ ret bfloat %res
+}
More information about the llvm-commits
mailing list