[llvm] [AMDGPU] Perform scalar FMUL with bf16 more efficiently (PR #213969)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 07:50:22 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: LU-JOHN
<details>
<summary>Changes</summary>
Perform scalar FMUL with bf16 more efficiently. Utilize v2bf16 patterns.
---
Full diff: https://github.com/llvm/llvm-project/pull/213969.diff
5 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+9-7)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll (+2-6)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll (+2-6)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e55356ab3ea24..189ec3fe1e3c1 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -247,6 +247,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// Widen scalar fcanonicalize to a v2bf16 operation with an unused high
// lane.
setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
+ // Widen scalar fmul to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FMUL, MVT::bf16, Custom);
}
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
@@ -7718,7 +7720,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
case ISD::UADDSAT:
@@ -7727,8 +7728,9 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
case ISD::FADD:
+ case ISD::FMUL:
if (Op.getValueType() == MVT::bf16)
- return lowerScalarBF16FAdd(Op, DAG);
+ return lowerScalarBF16BinaryOp(Op, DAG);
return splitBinaryVectorOp(Op, DAG);
case ISD::FCANONICALIZE:
if (Op.getValueType() == MVT::bf16)
@@ -8797,8 +8799,8 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
DAG.getTargetConstant(0, DL, MVT::i32));
}
-SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue SITargetLowering::lowerScalarBF16BinaryOp(SDValue Op,
+ SelectionDAG &DAG) const {
assert(Subtarget->hasBF16PackedInsts());
SDLoc DL(Op);
@@ -8815,10 +8817,10 @@ SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
SDValue LHS = WidenOperand(Op.getOperand(0));
SDValue RHS = WidenOperand(Op.getOperand(1));
- SDValue Add =
- DAG.getNode(ISD::FADD, DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+ SDValue Result =
+ DAG.getNode(Op.getOpcode(), DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Add,
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
DAG.getConstant(0, DL, MVT::i32));
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f358f59311c79..3e0e5da94471f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -166,7 +166,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
- SDValue lowerScalarBF16FAdd(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16BinaryOp(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerScalarBF16FCanonicalize(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index deb8f6e415b70..89006f14961eb 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -14442,7 +14442,7 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, 0 op_sel_hi:[1,1,0]
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fmul bfloat %a, %b
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
index 8de3847f1df8c..097adde1dece0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
@@ -15,11 +15,9 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; FAKE16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; FAKE16-NEXT: v_cos_bf16_e32 v0, v0
; FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
@@ -31,11 +29,9 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
; REAL16-NEXT: v_mov_b32_e32 v1, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
index ef331c8b0162d..d3eda06c9ddd1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
@@ -15,11 +15,9 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; FAKE16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; FAKE16-NEXT: v_sin_bf16_e32 v0, v0
; FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
@@ -31,11 +29,9 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
; REAL16-NEXT: v_mov_b32_e32 v1, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_sin_bf16_e32 v0.l, v0.l
; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
``````````
</details>
https://github.com/llvm/llvm-project/pull/213969
More information about the llvm-commits
mailing list