[llvm] [AMDGPU] Canonicalize scalar bf16 more efficiently (PR #213789)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 19:00:43 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/213789
>From 3d5cc5a4e29c20a903383a80cc970262a86781bb Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 3 Aug 2026 17:45:10 -0500
Subject: [PATCH 1/2] Canonicalize bf16 more efficiently
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 24 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/bf16.ll | 5 +-
.../AMDGPU/fcanonicalize-elimination.bf16.ll | 14 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 279 +++++-------------
5 files changed, 110 insertions(+), 213 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 612342b159583..f26f720c5dae6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -244,6 +244,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::FSUB, MVT::bf16, Expand);
// Widen scalar fadd to a v2bf16 operation with an unused high lane.
setOperationAction(ISD::FADD, MVT::bf16, Custom);
+ // Widen scalar fcanonicalize to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
}
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
@@ -7676,7 +7678,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ABS:
case ISD::FABS:
case ISD::FNEG:
- case ISD::FCANONICALIZE:
case ISD::BSWAP:
return splitUnaryVectorOp(Op, DAG);
case ISD::FP_TO_SINT_SAT:
@@ -7728,6 +7729,10 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
if (Op.getValueType() == MVT::bf16)
return lowerScalarBF16FAdd(Op, DAG);
return splitBinaryVectorOp(Op, DAG);
+ case ISD::FCANONICALIZE:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16FCanonicalize(Op, DAG);
+ return splitUnaryVectorOp(Op, DAG);
case ISD::FCOPYSIGN:
return lowerFCOPYSIGN(Op, DAG);
case ISD::MUL:
@@ -8816,6 +8821,23 @@ SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
DAG.getConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerScalarBF16FCanonicalize(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // Widen to v2bf16, canonicalize with v_pk_mul_bf16, then extract.
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+
+ SDValue Canonicalized =
+ DAG.getNode(ISD::FCANONICALIZE, DL, MVT::v2bf16, WideSrc, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Canonicalized,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 14f63325edde7..f358f59311c79 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -167,6 +167,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerScalarBF16FAdd(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16FCanonicalize(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 2ade9dd85ae3f..deb8f6e415b70 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -31324,10 +31324,7 @@ define bfloat @v_canonicalize_bf16(bfloat %a) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.canonicalize.bf16(bfloat %a)
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
index 6f74190238bf6..4d668540e308d 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
@@ -21,11 +21,8 @@ define bfloat @test_canonicalize_amdgcn_tanh_bf16(bfloat %a) {
; FAKE16-NEXT: s_wait_kmcnt 0x0
; FAKE16-NEXT: v_tanh_bf16_e32 v0, v0
; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; FAKE16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; REAL16-LABEL: test_canonicalize_amdgcn_tanh_bf16:
@@ -34,11 +31,8 @@ define bfloat @test_canonicalize_amdgcn_tanh_bf16(bfloat %a) {
; REAL16-NEXT: s_wait_kmcnt 0x0
; REAL16-NEXT: v_tanh_bf16_e32 v0.l, v0.l
; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%tanh = call bfloat @llvm.amdgcn.tanh.bf16(bfloat %a)
%canonicalized = call bfloat @llvm.canonicalize.bf16(bfloat %tanh)
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index e148ee9772ac4..660407c4850f9 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -51,14 +51,10 @@ define amdgpu_kernel void @v_test_canonicalize_var_bf16(ptr addrspace(1) %out) #
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, s0 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v[0:1], v0, off
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -74,13 +70,10 @@ define amdgpu_kernel void @s_test_canonicalize_var_bf16(ptr addrspace(1) %out, i
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshl_b32 s2, s2, 16
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_max_num_f32_e64 v0, s2, s2
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = bitcast i16 %val.arg to bfloat
%canonicalized = call bfloat @llvm.canonicalize.bf16(bfloat %val)
@@ -114,43 +107,21 @@ define <2 x bfloat> @v_test_canonicalize_build_vector_v2bf16(bfloat %lo, bfloat
define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %out) #1 {
-; FAKE16-LABEL: v_test_canonicalize_fabs_var_bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
-; FAKE16-NEXT: s_wait_loadcnt 0x0
-; FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; FAKE16-NEXT: s_endpgm
-;
-; REAL16-LABEL: v_test_canonicalize_fabs_var_bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; REAL16-NEXT: v_mov_b32_e32 v1, 0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: global_load_u16 v0, v1, s[0:1]
-; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_and_b32_e32 v0, 0x7fff, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: v_test_canonicalize_fabs_var_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
%val.fabs = call bfloat @llvm.fabs.bf16(bfloat %val)
%canonicalized = call bfloat @llvm.canonicalize.bf16(bfloat %val.fabs)
@@ -160,43 +131,21 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %o
define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(1) %out) #1 {
-; FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
-; FAKE16-NEXT: s_wait_loadcnt 0x0
-; FAKE16-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; FAKE16-NEXT: s_endpgm
-;
-; REAL16-LABEL: v_test_canonicalize_fneg_fabs_var_bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; REAL16-NEXT: v_mov_b32_e32 v1, 0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: global_load_u16 v0, v1, s[0:1]
-; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_or_b32_e32 v0, 0x8000, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: v_test_canonicalize_fneg_fabs_var_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
%val.fabs = call bfloat @llvm.fabs.bf16(bfloat %val)
%val.fabs.fneg = fneg bfloat %val.fabs
@@ -206,43 +155,21 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(
}
define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %out) #1 {
-; FAKE16-LABEL: v_test_canonicalize_fneg_var_bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
-; FAKE16-NEXT: s_wait_loadcnt 0x0
-; FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; FAKE16-NEXT: s_endpgm
-;
-; REAL16-LABEL: v_test_canonicalize_fneg_var_bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; REAL16-NEXT: v_mov_b32_e32 v1, 0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: global_load_u16 v0, v1, s[0:1]
-; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: v_test_canonicalize_fneg_var_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
%val.fneg = fneg bfloat %val
%canonicalized = call bfloat @llvm.canonicalize.bf16(bfloat %val.fneg)
@@ -251,43 +178,21 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %o
}
define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr addrspace(1) %out) #2 {
-; FAKE16-LABEL: v_test_no_denormals_canonicalize_fneg_var_bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
-; FAKE16-NEXT: s_wait_loadcnt 0x0
-; FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; FAKE16-NEXT: s_endpgm
-;
-; REAL16-LABEL: v_test_no_denormals_canonicalize_fneg_var_bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; REAL16-NEXT: v_mov_b32_e32 v1, 0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: global_load_u16 v0, v1, s[0:1]
-; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: v_test_no_denormals_canonicalize_fneg_var_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
%val.fneg = fneg bfloat %val
%canonicalized = call bfloat @llvm.canonicalize.bf16(bfloat %val.fneg)
@@ -296,43 +201,21 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr ad
}
define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(ptr addrspace(1) %out) #2 {
-; FAKE16-LABEL: v_test_no_denormals_canonicalize_fneg_fabs_var_bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; FAKE16-NEXT: v_nop
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
-; FAKE16-NEXT: s_wait_loadcnt 0x0
-; FAKE16-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v1
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; FAKE16-NEXT: s_endpgm
-;
-; REAL16-LABEL: v_test_no_denormals_canonicalize_fneg_fabs_var_bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
-; REAL16-NEXT: v_nop
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; REAL16-NEXT: v_mov_b32_e32 v1, 0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: global_load_u16 v0, v1, s[0:1]
-; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_or_b32_e32 v0, 0x8000, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
-; REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: v_test_no_denormals_canonicalize_fneg_fabs_var_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
%val.fabs = call bfloat @llvm.fabs.bf16(bfloat %val)
%val.fabs.fneg = fneg bfloat %val.fabs
>From 402f4440c6a79c2bd980f937f6895172c41d4e43 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 3 Aug 2026 19:23:17 -0500
Subject: [PATCH 2/2] Fix formatting
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f26f720c5dae6..e55356ab3ea24 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -244,7 +244,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::FSUB, MVT::bf16, Expand);
// Widen scalar fadd to a v2bf16 operation with an unused high lane.
setOperationAction(ISD::FADD, MVT::bf16, Custom);
- // Widen scalar fcanonicalize to a v2bf16 operation with an unused high lane.
+ // Widen scalar fcanonicalize to a v2bf16 operation with an unused high
+ // lane.
setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
}
@@ -8821,8 +8822,9 @@ SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
DAG.getConstant(0, DL, MVT::i32));
}
-SDValue SITargetLowering::lowerScalarBF16FCanonicalize(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+SITargetLowering::lowerScalarBF16FCanonicalize(SDValue Op,
+ SelectionDAG &DAG) const {
assert(Subtarget->hasBF16PackedInsts());
SDLoc DL(Op);
More information about the llvm-commits
mailing list