[llvm] [AMDGPU] Widen fminnum/fmaxnum bf16 to v2bf16 (PR #215042)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 18:03:15 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: LU-JOHN
<details>
<summary>Changes</summary>
Perform scalar FMINNUM/FMAXNUM with bf16 more efficiently. Utilize v2bf16 patterns.
---
Full diff: https://github.com/llvm/llvm-project/pull/215042.diff
3 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+11-13)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+2-8)
- (modified) llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll (+54-12)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1d165910adeba..d72e42cd3df97 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -247,13 +247,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
// path instead of promoting to f32.
setOperationAction(ISD::FSUB, MVT::bf16, Expand);
- // Widen scalar fadd to a v2bf16 operation with an unused high lane.
- setOperationAction(ISD::FADD, MVT::bf16, Custom);
- // Widen scalar fcanonicalize to a v2bf16 operation with an unused high
- // lane.
- setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
- // Widen scalar fmul to a v2bf16 operation with an unused high lane.
- setOperationAction(ISD::FMUL, MVT::bf16, Custom);
+ // Widen scalar operations to a v2bf16 operation with an unused high lane.
+ setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
+ ISD::FCANONICALIZE},
+ MVT::bf16, Custom);
}
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
@@ -7717,9 +7714,15 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Op.getOperand(0).getValueType().getScalarType() == MVT::f32)
return splitUnaryVectorOp(Op, DAG);
return LowerFP_TO_INT_SAT(Op, DAG);
+ case ISD::FADD:
+ case ISD::FMUL:
case ISD::FMINNUM:
case ISD::FMAXNUM:
- return lowerFMINNUM_FMAXNUM(Op, DAG);
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16BinaryOp(Op, DAG);
+ if (Op.getOpcode() == ISD::FMINNUM || Op.getOpcode() == ISD::FMAXNUM)
+ return lowerFMINNUM_FMAXNUM(Op, DAG);
+ return splitBinaryVectorOp(Op, DAG);
case ISD::FMINIMUMNUM:
case ISD::FMAXIMUMNUM:
return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
@@ -7755,11 +7758,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SADDSAT:
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
- case ISD::FADD:
- case ISD::FMUL:
- if (Op.getValueType() == MVT::bf16)
- return lowerScalarBF16BinaryOp(Op, DAG);
- return splitBinaryVectorOp(Op, DAG);
case ISD::FCANONICALIZE:
if (Op.getValueType() == MVT::bf16)
return lowerScalarBF16FCanonicalize(Op, DAG);
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 4d17ca1d21db5..e8536b7edb5c4 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -19217,10 +19217,7 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_min_num_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
ret bfloat %op
@@ -23715,10 +23712,7 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll b/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
index ed5989352c77e..a09a0f2da9e5a 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
@@ -485,7 +485,7 @@ define double @v_no_max3_maxnum_tree4_f64(double %a, double %b, double %c, doubl
ret double %result
}
-; Negative test: bf16 is promoted to f32 with conversions, tree combine cannot apply
+; Negative test: bf16 has no max3 on any target yet, tree combine must not fire
define bfloat @v_no_max3_maxnum_tree4_bf16(bfloat %a, bfloat %b, bfloat %c, bfloat %d) {
; GFX9-LABEL: v_no_max3_maxnum_tree4_bf16:
; GFX9: ; %bb.0:
@@ -522,17 +522,10 @@ define bfloat @v_no_max3_maxnum_tree4_bf16(bfloat %a, bfloat %b, bfloat %c, bflo
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_max_num_f32 v2, v2, v3 :: v_dual_max_num_f32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
+; GFX1250-NEXT: v_pk_max_num_bf16 v1, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%max.ab = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
%max.cd = call bfloat @llvm.maxnum.bf16(bfloat %c, bfloat %d)
@@ -540,6 +533,54 @@ define bfloat @v_no_max3_maxnum_tree4_bf16(bfloat %a, bfloat %b, bfloat %c, bflo
ret bfloat %result
}
+; Negative test: bf16 has no min3 on any target yet, tree combine must not fire
+define bfloat @v_no_min3_minnum_tree4_bf16(bfloat %a, bfloat %b, bfloat %c, bfloat %d) {
+; GFX9-LABEL: v_no_min3_minnum_tree4_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v1, v2, v1
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: v_no_min3_minnum_tree4_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-NEXT: v_pk_min_num_bf16 v1, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_min_num_bf16 v0, v0, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %min.ab = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ %min.cd = call bfloat @llvm.minnum.bf16(bfloat %c, bfloat %d)
+ %result = call bfloat @llvm.minnum.bf16(bfloat %min.ab, bfloat %min.cd)
+ ret bfloat %result
+}
+
; Two-level ternary tree
define float @v_max3_maxnum_ternary_2level_f32(
; GFX9-LABEL: v_max3_maxnum_ternary_2level_f32:
@@ -763,5 +804,6 @@ declare float @llvm.minimum.f32(float, float)
declare half @llvm.maxnum.f16(half, half)
declare double @llvm.maxnum.f64(double, double)
declare bfloat @llvm.maxnum.bf16(bfloat, bfloat)
+declare bfloat @llvm.minnum.bf16(bfloat, bfloat)
declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>)
declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>)
``````````
</details>
https://github.com/llvm/llvm-project/pull/215042
More information about the llvm-commits
mailing list