[llvm] ac067a7 - [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on gfx1250/gfx13 (#209128)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 23:49:07 PDT 2026
Author: Harrison Hao
Date: 2026-07-16T14:49:03+08:00
New Revision: ac067a7e1366f1a9fca74b2cb69d6b005c6b732e
URL: https://github.com/llvm/llvm-project/commit/ac067a7e1366f1a9fca74b2cb69d6b005c6b732e
DIFF: https://github.com/llvm/llvm-project/commit/ac067a7e1366f1a9fca74b2cb69d6b005c6b732e.diff
LOG: [AMDGPU] Use v_pk_add_bf16 for scalar bf16 fadd on gfx1250/gfx13 (#209128)
Targets with packed bf16 instructions can do a scalar bf16 fadd with a
single `v_pk_add_bf16` operating on the low half, instead of promoting
to f32 and going through `v_fma_mix_f32_bf16 + v_cvt_pk_bf16_f32`.
Added:
Modified:
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/AMDGPU/SIISelLowering.h
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/bf16.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9cf33b4ccbb87..cfd09eda8f11b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -236,6 +236,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Widen scalar fadd to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FADD, MVT::bf16, Custom);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
@@ -7682,7 +7691,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FADD:
case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
@@ -7691,6 +7699,10 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SADDSAT:
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
+ case ISD::FADD:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16FAdd(Op, DAG);
+ return splitBinaryVectorOp(Op, DAG);
case ISD::FCOPYSIGN:
return lowerFCOPYSIGN(Op, DAG);
case ISD::MUL:
@@ -8750,6 +8762,31 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
DAG.getTargetConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+
+ auto WidenOperand = [&](SDValue Src) {
+ if (Src.getOpcode() == ISD::FNEG) {
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16,
+ Src.getOperand(0));
+ return DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, WideSrc, Src->getFlags());
+ }
+
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+ };
+
+ SDValue LHS = WidenOperand(Op.getOperand(0));
+ SDValue RHS = WidenOperand(Op.getOperand(1));
+ SDValue Add =
+ DAG.getNode(ISD::FADD, DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Add,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index a181fa072d534..86653fe6920c5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -163,6 +163,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16FAdd(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 25991d274a278..ebd7f2b7b60da 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15625,195 +15625,96 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
;
-; GFX1364-TRUE16-LABEL: uniform_fadd_bf16:
-; GFX1364-TRUE16: ; %bb.0:
-; GFX1364-TRUE16-NEXT: s_clause 0x1
-; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-TRUE16-NEXT: s_not_b32 s10, s2
-; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1364-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1364-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1364-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1364-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
-; GFX1364-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1364-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1364-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1364-TRUE16-NEXT: s_endpgm
-;
-; GFX1364-FAKE16-LABEL: uniform_fadd_bf16:
-; GFX1364-FAKE16: ; %bb.0:
-; GFX1364-FAKE16-NEXT: s_clause 0x1
-; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1364-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1364-FAKE16-NEXT: s_not_b32 s10, s2
-; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1364-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1364-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1364-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1364-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX1364-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1364-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1364-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1364-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1364-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1364-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
-; GFX1364-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1364-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1364-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1364-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
-; GFX1364-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1364-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1364-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1364-FAKE16-NEXT: s_endpgm
-;
-; GFX1332-TRUE16-LABEL: uniform_fadd_bf16:
-; GFX1332-TRUE16: ; %bb.0:
-; GFX1332-TRUE16-NEXT: s_clause 0x1
-; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1332-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s9
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
-; GFX1332-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1332-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
-; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
-; GFX1332-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1332-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1332-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1332-TRUE16-NEXT: s_endpgm
+; GFX1364-LABEL: uniform_fadd_bf16:
+; GFX1364: ; %bb.0:
+; GFX1364-NEXT: s_clause 0x1
+; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1364-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1364-NEXT: s_mov_b32 s6, -1
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: s_and_b64 s[4:5], s[2:3], -4
+; GFX1364-NEXT: s_and_b32 s2, s2, 3
+; GFX1364-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1364-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1364-NEXT: s_lshl_b32 s2, 0xffff, s9
+; GFX1364-NEXT: s_not_b32 s10, s2
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: s_mov_b64 s[2:3], 0
+; GFX1364-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1364-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1364-NEXT: v_pk_add_bf16 v0, v0, s8
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1364-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: v_mov_b32_e32 v2, v0
+; GFX1364-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1364-NEXT: s_wait_loadcnt 0x0
+; GFX1364-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1364-NEXT: v_mov_b32_e32 v1, v2
+; GFX1364-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1364-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1364-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1364-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1364-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1364-NEXT: s_mov_b32 s2, -1
+; GFX1364-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1364-NEXT: s_endpgm
;
-; GFX1332-FAKE16-LABEL: uniform_fadd_bf16:
-; GFX1332-FAKE16: ; %bb.0:
-; GFX1332-FAKE16-NEXT: s_clause 0x1
-; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s9, 0
-; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
-; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, s6
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1332-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
-; GFX1332-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1332-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1332-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1332-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1332-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1332-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1332-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1332-FAKE16-NEXT: s_or_b32 s9, vcc_lo, s9
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
-; GFX1332-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
-; GFX1332-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1332-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s9
-; GFX1332-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
-; GFX1332-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1332-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1332-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX1332-FAKE16-NEXT: s_endpgm
+; GFX1332-LABEL: uniform_fadd_bf16:
+; GFX1332: ; %bb.0:
+; GFX1332-NEXT: s_clause 0x1
+; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1332-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
+; GFX1332-NEXT: s_mov_b32 s9, 0
+; GFX1332-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1332-NEXT: s_wait_kmcnt 0x0
+; GFX1332-NEXT: s_and_b64 s[4:5], s[2:3], -4
+; GFX1332-NEXT: s_and_b32 s2, s2, 3
+; GFX1332-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1332-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1332-NEXT: s_not_b32 s3, s3
+; GFX1332-NEXT: s_wait_kmcnt 0x0
+; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s6, -1
+; GFX1332-NEXT: .LBB19_1: ; %atomicrmw.start
+; GFX1332-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1332-NEXT: v_pk_add_bf16 v0, v0, s8
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1332-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1332-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1332-NEXT: s_wait_loadcnt 0x0
+; GFX1332-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1332-NEXT: v_mov_b32_e32 v1, v2
+; GFX1332-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1332-NEXT: s_cbranch_execnz .LBB19_1
+; GFX1332-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1332-NEXT: v_lshrrev_b32_e32 v0, s2, v2
+; GFX1332-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s2, -1
+; GFX1332-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1332-NEXT: s_endpgm
%rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
store bfloat %rmw, ptr addrspace(1) %result
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index d09ec87e1a19f..c595e6a15d287 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9258,9 +9258,7 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fadd bfloat %a, %b
ret bfloat %op
@@ -13404,10 +13402,7 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 1.0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 1.0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
@@ -13515,10 +13510,7 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s0, 0x42280000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s0 op_sel:[0,1,0] op_sel_hi:[1,1,0]
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_add_bf16 v0, 0x4228, v0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 42.0
ret bfloat %add
@@ -13634,9 +13626,7 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v1, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fsub bfloat %a, %b
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 72e06145dc850..74cf3e6b49493 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13673,95 +13673,49 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v3, v0
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB54_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -14260,95 +14214,49 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB55_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -14867,97 +14775,50 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB56_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -15477,93 +15338,47 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v3, v0
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB57_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -16068,12 +15883,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16112,12 +15925,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16661,12 +16472,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -16706,12 +16515,10 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
@@ -17246,11 +17053,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v3, v5, v2
+; GFX1250-TRUE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17280,11 +17084,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
+; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -17726,11 +17527,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v3, 1.0, v4 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4
+; GFX1250-TRUE16-NEXT: v_bfi_b32 v2, 0xffff0000, v3, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17758,12 +17556,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
+; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -18180,95 +17975,49 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX1250-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v6, v7, v4, v5
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v4, v4
+; GFX1250-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v7, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT: v_pk_add_bf16 v5, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB62_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12-TRUE16: ; %bb.0:
@@ -18813,12 +18562,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
+; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
+; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18857,12 +18604,10 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 94e9ce55946ab..121175707a47f 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -531,3 +531,52 @@ define <4 x bfloat> @v_maxnum_v4bf16_neg1(<4 x bfloat> %a, <4 x bfloat> %b) {
%max = call <4 x bfloat> @llvm.maxnum.v4bf16( <4 x bfloat> %a, <4 x bfloat> %neg)
ret <4 x bfloat> %max
}
+
+define bfloat @v_fadd_bf16_neg(bfloat %a, bfloat %b) {
+; GFX1250-LABEL: v_fadd_bf16_neg:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_neg:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %neg = fneg bfloat %b
+ %res = fadd bfloat %a, %neg
+ ret bfloat %res
+}
+
+define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
+; GFX1250-LABEL: v_fadd_bf16_chain:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: v_fadd_bf16_chain:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1310-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-NEXT: v_pk_add_bf16 v0, v0, v2
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %num = fadd bfloat %a, %b
+ %res = fadd bfloat %num, %c
+ ret bfloat %res
+}
More information about the llvm-commits
mailing list