[llvm] [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on gfx1250/gfx13 (PR #209128)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 19:33:10 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/209128
>From b563449fa89f079f82b452ee9b55bb61eb27172d Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 13 Jul 2026 17:59:28 +0800
Subject: [PATCH 1/2] [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on
gfx1250/gfx13
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 9 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 +++
llvm/lib/Target/AMDGPU/SIInstructions.td | 12 +-
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 6 +-
.../atomic_optimizations_global_pointer.ll | 64 +++++------
llvm/test/CodeGen/AMDGPU/bf16.ll | 90 +++++++++------
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 107 +++++++-----------
.../CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll | 49 ++++++++
8 files changed, 209 insertions(+), 141 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4b2fd87c5b078..7129a3738eab8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -236,6 +236,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Use v_pk_add_bf16 for scalar fadd.
+ setOperationAction(ISD::FADD, MVT::bf16, Legal);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 81b7bf7f769ad..da7a8358b967e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2416,6 +2416,19 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
break;
}
+ case AMDGPU::V_ADD_BF16_PSEUDO_e64: {
+ assert(ST.hasBF16PackedInsts());
+ MI.setDesc(get(AMDGPU::V_PK_ADD_BF16));
+ MI.addOperand(MachineOperand::CreateImm(0)); // op_sel
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo
+ MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi
+ MachineOperand *Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
+ MachineOperand *Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
+ break;
+ }
+
case AMDGPU::GET_STACK_BASE:
// The stack starts at offset 0 unless we need to reserve some space at the
// bottom.
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d3ea76ebb647d..e252f4b8a26cf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2113,7 +2113,7 @@ def : ClampPat<V_MAX_F16_fake16_e64, f16>;
let True16Predicate = UseFakeTrue16Insts in
def : ClampPat<V_MAX_BF16_PSEUDO_e64, bf16>;
let True16Predicate = UseRealTrue16Insts,
- SubtargetPredicate = HasBF16PackedInsts in
+ SubtargetPredicate = HasBF16PackedInsts in {
def : GCNPat<(bf16 (AMDGPUclamp (VOP3Mods bf16:$src0, i32:$src0_modifiers))),
(EXTRACT_SUBREG
(V_PK_MAX_NUM_BF16
@@ -2121,6 +2121,16 @@ let True16Predicate = UseRealTrue16Insts,
$src0_modifiers, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
DSTCLAMP.ENABLE), lo16)
>;
+ def : GCNPat<
+ (bf16 (fadd (bf16 (VOP3Mods bf16:$src0, i32:$src0_mods)),
+ (bf16 (VOP3Mods bf16:$src1, i32:$src1_mods)))),
+ (EXTRACT_SUBREG
+ (V_PK_ADD_BF16
+ $src0_mods, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ $src1_mods, (REG_SEQUENCE VGPR_32, $src1, lo16, (bf16 (IMPLICIT_DEF)), hi16),
+ DSTCLAMP.NONE), lo16)
+ >;
+}
let SubtargetPredicate = HasVOP3PInsts in {
def : GCNPat <
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index f99367d198f26..b64d9f3e13fac 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1458,8 +1458,10 @@ let isCommutable = 1, isReMaterializable = 1 in {
// Scalar pseudo used to emulate AMDGPUClamp.
// Expanded to V_PK_MAX_NUM_BF16 with unused high half.
// FIXME-TRUE16: Pseudo expansion of this won't work with True16.
- let True16Predicate = UseFakeTrue16Insts in
- defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ let True16Predicate = UseFakeTrue16Insts in {
+ defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
+ defm V_ADD_BF16_PSEUDO : VOP3Inst <"v_add_bf16", VOP_BF16_BF16_BF16, any_fadd>;
+ }
}
let SubtargetPredicate = HasPackedFP64Ops, SchedRW = [WriteDouble] in {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index cb6e860c92cc1..9feeb3ce0dbcc 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15629,33 +15629,31 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16: ; %bb.0:
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1364-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1364-TRUE16-NEXT: s_not_b32 s9, s2
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, s3
; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1364-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15667,7 +15665,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1364-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX1364-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
; GFX1364-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1364-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -15695,15 +15693,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1364-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1364-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -15725,43 +15721,41 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-TRUE16: ; %bb.0:
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1332-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1332-TRUE16-NEXT: s_not_b32 s3, s3
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1332-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1332-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1332-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1332-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -15790,14 +15784,12 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1332-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1332-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-FAKE16-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index dc125ec5840ae..87ddda1ed5f40 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9254,14 +9254,19 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fadd bfloat %a, %b
ret bfloat %op
}
@@ -13400,15 +13405,21 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 1.0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x3f80
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, 1.0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
}
@@ -13511,15 +13522,21 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fadd_bf16_fpimm_1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 0x42280000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4228
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, 0x4228, v0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 42.0
ret bfloat %add
}
@@ -13630,14 +13647,19 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_fsub_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v1, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_fsub_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_fsub_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = fsub bfloat %a, %b
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index e385261b9bf8a..1d95592c8ae53 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13693,12 +13693,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -13738,12 +13737,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14280,12 +14278,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14325,12 +14322,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14888,12 +14884,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14934,12 +14929,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -15496,12 +15490,10 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v4, v4, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -15540,12 +15532,10 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16068,12 +16058,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16112,12 +16100,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16661,12 +16647,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16706,12 +16690,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17246,10 +17228,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v3, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17280,10 +17261,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17726,10 +17706,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v3, 1.0, v4 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17758,11 +17737,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -18200,12 +18177,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18245,12 +18221,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18813,12 +18788,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18857,12 +18830,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 03897e4ee9a24..08daa1a23778c 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -531,3 +531,52 @@ define <4 x bfloat> @v_maxnum_v4bf16_neg1(<4 x bfloat> %a, <4 x bfloat> %b) {
%max = call <4 x bfloat> @llvm.maxnum.v4bf16( <4 x bfloat> %a, <4 x bfloat> %neg)
ret <4 x bfloat> %max
}
+
+define bfloat @v_fadd_bf16_neg(bfloat %a, bfloat %b) {
+; GFX1250-LABEL: v_fadd_bf16_neg:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_neg:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %neg = fneg bfloat %b
+ %res = fadd bfloat %a, %neg
+ ret bfloat %res
+}
+
+define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
+; GFX1250-LABEL: v_fadd_bf16_chain:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_chain:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1310-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %num = fadd bfloat %a, %b
+ %res = fadd bfloat %num, %c
+ ret bfloat %res
+}
>From 906e54ac0e90144b22c1c73836b9c82d82946e6c Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 14 Jul 2026 10:32:31 +0800
Subject: [PATCH 2/2] Implement in legalizer
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 34 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 13 -
llvm/lib/Target/AMDGPU/SIInstructions.td | 12 +-
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 6 +-
.../atomic_optimizations_global_pointer.ll | 269 +++----
llvm/test/CodeGen/AMDGPU/bf16.ll | 80 +--
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 680 ++++++------------
.../CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll | 8 +-
9 files changed, 379 insertions(+), 724 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 7129a3738eab8..a3238386fcceb 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -241,8 +241,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
// path instead of promoting to f32.
setOperationAction(ISD::FSUB, MVT::bf16, Expand);
- // Use v_pk_add_bf16 for scalar fadd.
- setOperationAction(ISD::FADD, MVT::bf16, Legal);
+ // Widen scalar fadd to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FADD, MVT::bf16, Custom);
}
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
@@ -7663,7 +7663,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FADD:
case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
@@ -7672,6 +7671,10 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SADDSAT:
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
+ case ISD::FADD:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16FAdd(Op, DAG);
+ return splitBinaryVectorOp(Op, DAG);
case ISD::FCOPYSIGN:
return lowerFCOPYSIGN(Op, DAG);
case ISD::MUL:
@@ -8707,6 +8710,31 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
DAG.getTargetConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+
+ auto WidenOperand = [&](SDValue Src) {
+ if (Src.getOpcode() == ISD::FNEG) {
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16,
+ Src.getOperand(0));
+ return DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, WideSrc, Src->getFlags());
+ }
+
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+ };
+
+ SDValue LHS = WidenOperand(Op.getOperand(0));
+ SDValue RHS = WidenOperand(Op.getOperand(1));
+ SDValue Add =
+ DAG.getNode(ISD::FADD, DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Add,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index a181fa072d534..86653fe6920c5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -163,6 +163,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16FAdd(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index da7a8358b967e..81b7bf7f769ad 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2416,19 +2416,6 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
break;
}
- case AMDGPU::V_ADD_BF16_PSEUDO_e64: {
- assert(ST.hasBF16PackedInsts());
- MI.setDesc(get(AMDGPU::V_PK_ADD_BF16));
- MI.addOperand(MachineOperand::CreateImm(0)); // op_sel
- MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo
- MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi
- MachineOperand *Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
- Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
- MachineOperand *Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
- Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
- break;
- }
-
case AMDGPU::GET_STACK_BASE:
// The stack starts at offset 0 unless we need to reserve some space at the
// bottom.
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e252f4b8a26cf..d3ea76ebb647d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2113,7 +2113,7 @@ def : ClampPat<V_MAX_F16_fake16_e64, f16>;
let True16Predicate = UseFakeTrue16Insts in
def : ClampPat<V_MAX_BF16_PSEUDO_e64, bf16>;
let True16Predicate = UseRealTrue16Insts,
- SubtargetPredicate = HasBF16PackedInsts in {
+ SubtargetPredicate = HasBF16PackedInsts in
def : GCNPat<(bf16 (AMDGPUclamp (VOP3Mods bf16:$src0, i32:$src0_modifiers))),
(EXTRACT_SUBREG
(V_PK_MAX_NUM_BF16
@@ -2121,16 +2121,6 @@ let True16Predicate = UseRealTrue16Insts,
$src0_modifiers, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
DSTCLAMP.ENABLE), lo16)
>;
- def : GCNPat<
- (bf16 (fadd (bf16 (VOP3Mods bf16:$src0, i32:$src0_mods)),
- (bf16 (VOP3Mods bf16:$src1, i32:$src1_mods)))),
- (EXTRACT_SUBREG
- (V_PK_ADD_BF16
- $src0_mods, (REG_SEQUENCE VGPR_32, $src0, lo16, (bf16 (IMPLICIT_DEF)), hi16),
- $src1_mods, (REG_SEQUENCE VGPR_32, $src1, lo16, (bf16 (IMPLICIT_DEF)), hi16),
- DSTCLAMP.NONE), lo16)
- >;
-}
let SubtargetPredicate = HasVOP3PInsts in {
def : GCNPat <
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index b64d9f3e13fac..f99367d198f26 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1458,10 +1458,8 @@ let isCommutable = 1, isReMaterializable = 1 in {
// Scalar pseudo used to emulate AMDGPUClamp.
// Expanded to V_PK_MAX_NUM_BF16 with unused high half.
// FIXME-TRUE16: Pseudo expansion of this won't work with True16.
- let True16Predicate = UseFakeTrue16Insts in {
- defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
- defm V_ADD_BF16_PSEUDO : VOP3Inst <"v_add_bf16", VOP_BF16_BF16_BF16, any_fadd>;
- }
+ let True16Predicate = UseFakeTrue16Insts in
+ defm V_MAX_BF16_PSEUDO : VOP3Inst <"v_max_bf16", VOP_BF16_BF16_BF16>;
}
let SubtargetPredicate = HasPackedFP64Ops, SchedRW = [WriteDouble] in {
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 9feeb3ce0dbcc..f28053c537dba 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15625,187 +15625,96 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
;
-; GFX1364-TRUE16-LABEL: uniform_fadd_bf16:
-; GFX1364-TRUE16: ; %bb.0:
-; GFX1364-TRUE16-NEXT: s_clause 0x1
-; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1364-TRUE16-NEXT: s_not_b32 s9, s2
-; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
-; GFX1364-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1364-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1364-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1364-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1364-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
-; GFX1364-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1364-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1364-TRUE16-NEXT: s_endpgm
-;
-; GFX1364-FAKE16-LABEL: uniform_fadd_bf16:
-; GFX1364-FAKE16: ; %bb.0:
-; GFX1364-FAKE16-NEXT: s_clause 0x1
-; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-FAKE16-NEXT: s_not_b32 s10, s2
-; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1364-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1364-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1364-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1364-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX1364-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1364-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1364-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1364-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
-; GFX1364-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1364-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1364-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
-; GFX1364-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1364-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1364-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1364-FAKE16-NEXT: s_endpgm
-;
-; GFX1332-TRUE16-LABEL: uniform_fadd_bf16:
-; GFX1332-TRUE16: ; %bb.0:
-; GFX1332-TRUE16-NEXT: s_clause 0x1
-; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v4.l, s6
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1332-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_pk_add_bf16 v0, v0, v4 op_sel_hi:[0,0]
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1332-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
-; GFX1332-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1332-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
-; GFX1332-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1332-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1332-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1332-TRUE16-NEXT: s_endpgm
+; GFX1364-LABEL: uniform_fadd_bf16:
+; GFX1364: ; %bb.0:
+; GFX1364-NEXT: s_clause 0x1
+; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1364-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1364-NEXT: s_mov_b32 s6, -1
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: s_and_b64 s[4:5], s[2:3], -4
+; GFX1364-NEXT: s_and_b32 s2, s2, 3
+; GFX1364-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1364-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1364-NEXT: s_lshl_b32 s2, 0xffff, s9
+; GFX1364-NEXT: s_not_b32 s10, s2
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: s_mov_b64 s[2:3], 0
+; GFX1364-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1364-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1364-NEXT: v_pk_add_bf16 v0, v0, s8
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1364-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: v_mov_b32_e32 v2, v0
+; GFX1364-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1364-NEXT: s_wait_loadcnt 0x0
+; GFX1364-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1364-NEXT: v_mov_b32_e32 v1, v2
+; GFX1364-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1364-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1364-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1364-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1364-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1364-NEXT: s_mov_b32 s2, -1
+; GFX1364-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1364-NEXT: s_endpgm
;
-; GFX1332-FAKE16-LABEL: uniform_fadd_bf16:
-; GFX1332-FAKE16: ; %bb.0:
-; GFX1332-FAKE16-NEXT: s_clause 0x1
-; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1332-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1332-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-FAKE16-NEXT: v_pk_add_bf16 v0, v0, s8
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1332-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1332-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1332-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1332-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-FAKE16-NEXT: s_or_b32 s9, vcc_lo, s9
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
-; GFX1332-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1332-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
-; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
-; GFX1332-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1332-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1332-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1332-FAKE16-NEXT: s_endpgm
+; GFX1332-LABEL: uniform_fadd_bf16:
+; GFX1332: ; %bb.0:
+; GFX1332-NEXT: s_clause 0x1
+; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1332-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1332-NEXT: s_mov_b32 s9, 0
+; GFX1332-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1332-NEXT: s_wait_kmcnt 0x0
+; GFX1332-NEXT: s_and_b64 s[4:5], s[2:3], -4
+; GFX1332-NEXT: s_and_b32 s2, s2, 3
+; GFX1332-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1332-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1332-NEXT: s_not_b32 s3, s3
+; GFX1332-NEXT: s_wait_kmcnt 0x0
+; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s6, -1
+; GFX1332-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1332-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1332-NEXT: v_pk_add_bf16 v0, v0, s8
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1332-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1332-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1332-NEXT: s_wait_loadcnt 0x0
+; GFX1332-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1332-NEXT: v_mov_b32_e32 v1, v2
+; GFX1332-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1332-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1332-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1332-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1332-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s2, -1
+; GFX1332-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1332-NEXT: s_endpgm
%rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
store bfloat %rmw, ptr addrspace(1) %result
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 87ddda1ed5f40..9ea63a57e1df3 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9254,19 +9254,12 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_fadd_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_fadd_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fadd_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fadd bfloat %a, %b
ret bfloat %op
}
@@ -13405,21 +13398,12 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_0:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x3f80
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_0:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, 1.0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fadd_bf16_fpimm_0:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 1.0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
}
@@ -13522,21 +13506,12 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_fadd_bf16_fpimm_1:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4228
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_fadd_bf16_fpimm_1:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, 0x4228, v0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fadd_bf16_fpimm_1:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, 0x4228, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 42.0
ret bfloat %add
}
@@ -13647,19 +13622,12 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_fsub_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_fsub_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fsub_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fsub bfloat %a, %b
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 1d95592c8ae53..bb97084991c13 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13673,93 +13673,49 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v3, v0
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB54_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -14258,93 +14214,49 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB55_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -14863,95 +14775,50 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB56_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -15471,89 +15338,47 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v4, v4, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v3, v0
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB57_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -16058,8 +15883,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16647,8 +16472,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -17228,10 +17053,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v3, v5, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v3, v5, v2
+; GFX1250-TRUE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17262,9 +17085,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -17706,10 +17527,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4
+; GFX1250-TRUE16-NEXT: v_bfi_b32 v2, 0xffff0000, v3, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17738,9 +17557,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -18157,93 +17975,49 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v5, v5, v2 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v5, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB62_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -18788,8 +18562,8 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6 op_sel_hi:[0,0]
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 08daa1a23778c..96769d9e1b159 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -537,7 +537,7 @@ define bfloat @v_fadd_bf16_neg(bfloat %a, bfloat %b) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0] neg_lo:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-LABEL: v_fadd_bf16_neg:
@@ -547,7 +547,7 @@ define bfloat @v_fadd_bf16_neg(bfloat %a, bfloat %b) {
; GFX1310-NEXT: s_wait_samplecnt 0x0
; GFX1310-NEXT: s_wait_bvhcnt 0x0
; GFX1310-NEXT: s_wait_kmcnt 0x0
-; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1]
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX1310-NEXT: s_set_pc_i64 s[30:31]
%neg = fneg bfloat %b
%res = fadd bfloat %a, %neg
@@ -559,10 +559,10 @@ define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 op_sel_hi:[0,0]
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-LABEL: v_fadd_bf16_chain:
More information about the llvm-commits
mailing list