[llvm] [AMDGPU] Code clean up and denorm check removal for bf16 omod folding (PR #219042)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 12:08:10 PDT 2026
https://github.com/changpeng updated https://github.com/llvm/llvm-project/pull/219042
>From cc11f92493ffd8b99dcc2477b8cfcddaf5bf0b6c Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Wed, 26 Aug 2026 14:03:44 -0700
Subject: [PATCH 1/3] [AMDGPU] Code clean up and denorm check removal for bf16
omod folding
This is the follow-up PR to address the additional comments and
suggestions from https://github.com/llvm/llvm-project/pull/218286
V_PK_MUL_BF16 and V_PK_ADD_BF16 ignore the DENORM mode controls and
always preserve denormals, so the output denormal mode has no bearing on
whether an omod can be folded into them. Drop the
FP32Denormals.Output == PreserveSign check from both cases in isOMod()
and document why it is not needed.
Also clean up the surrounding code:
- src0_modifiers/src1_modifiers are always present on these packed
opcodes (VOP3P_Profile sets HasModifiers), so drop the null checks and
dereference the operands directly.
- Fix the comment on the modifier check: rather than "modifiers other
than op_sel_hi block folding", the point is that when omod is applied
to a packed instruction it only applies to the low half of the input
and output.
- In tryFoldOMod(), drop the redundant isReg()/isImm() checks when
looking through a REG_SEQUENCE; a 5-operand REG_SEQUENCE always has
that operand shape.
Since the denormal mode no longer matters, the tests in
trans-bf16-omod.ll no longer need the denormal_fpenv(preservesign)
attribute. The checked assembly is unchanged.
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 20 ++++++++---------
llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll | 24 ++++++++++-----------
2 files changed, 20 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 9d1fc8deb9356..383da8bc396bf 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2505,8 +2505,9 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
}
case AMDGPU::V_PK_MUL_BF16: {
// OMOD folding for BF16 packed multiply
- if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign ||
- MI.mayRaiseFPException())
+ // PK_*_BF16 ignores DENORM controls, and always preserves denorms
+
+ if (MI.mayRaiseFPException())
return {nullptr, SIOutMods::NONE};
const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
@@ -2527,8 +2528,8 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
const MachineOperand *Src1Mods =
TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
- if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
- (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
+ (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
return {nullptr, SIOutMods::NONE};
@@ -2537,8 +2538,7 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
}
case AMDGPU::V_PK_ADD_BF16: {
// OMOD folding for BF16 packed add: x + x -> x * 2
- if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign)
- return {nullptr, SIOutMods::NONE};
+ // PK_*_BF16 ignores DENORM controls, and always preserves denorms
const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
@@ -2552,8 +2552,8 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
const MachineOperand *Src1Mods =
TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
- if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
- (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
+ (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
return {nullptr, SIOutMods::NONE};
@@ -2581,9 +2581,7 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
// In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
// REG_SEQUENCE. Look through it to find the actual instruction.
if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
- Def->getOperand(1).isReg() && Def->getOperand(2).isImm() &&
- Def->getOperand(2).getImm() == AMDGPU::lo16 &&
- Def->getOperand(3).isReg()) {
+ Def->getOperand(2).getImm() == AMDGPU::lo16) {
// Only look through if the high 16 bits are undefined
bool CanLookThrough = true;
MachineInstr *Hi16Def = MRI->getVRegDef(Def->getOperand(3).getReg());
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
index dba56ca84f0a1..4bb5ab6a33592 100644
--- a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -6,7 +6,7 @@
; FIXME: GlobalISel does not work with bf16
-define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -34,7 +34,7 @@ define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -62,7 +62,7 @@ define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_log_bf16_div2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -90,7 +90,7 @@ define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -118,7 +118,7 @@ define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %o
ret void
}
-define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -146,7 +146,7 @@ define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %o
ret void
}
-define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_log_bf16_div2_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -175,7 +175,7 @@ define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %o
}
-define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_rcp_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -203,7 +203,7 @@ define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_rsq_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -231,7 +231,7 @@ define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_sin_bf16_div2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -259,7 +259,7 @@ define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_sqrt_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -287,7 +287,7 @@ define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
-define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
; SDAG-FAKE16-LABEL: v_tanh_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -314,5 +314,3 @@ define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
store bfloat %mul4, ptr addrspace(1) %out
ret void
}
-
-attributes #0 = { nounwind denormal_fpenv(preservesign) }
>From 4a157adfa910cfb2147d83e125d6a05cc2dc9798 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Fri, 28 Aug 2026 11:14:54 -0700
Subject: [PATCH 2/3] [AMDGPU] Perform omod folding only when denormal flushing
is enabled
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 8 +++----
llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll | 24 +++++++++++----------
2 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 383da8bc396bf..1f0750d66684c 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2505,9 +2505,8 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
}
case AMDGPU::V_PK_MUL_BF16: {
// OMOD folding for BF16 packed multiply
- // PK_*_BF16 ignores DENORM controls, and always preserves denorms
-
- if (MI.mayRaiseFPException())
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign ||
+ MI.mayRaiseFPException())
return {nullptr, SIOutMods::NONE};
const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
@@ -2538,7 +2537,8 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
}
case AMDGPU::V_PK_ADD_BF16: {
// OMOD folding for BF16 packed add: x + x -> x * 2
- // PK_*_BF16 ignores DENORM controls, and always preserves denorms
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign)
+ return {nullptr, SIOutMods::NONE};
const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
index 4bb5ab6a33592..dba56ca84f0a1 100644
--- a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -6,7 +6,7 @@
; FIXME: GlobalISel does not work with bf16
-define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -34,7 +34,7 @@ define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -62,7 +62,7 @@ define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_log_bf16_div2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -90,7 +90,7 @@ define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -118,7 +118,7 @@ define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %o
ret void
}
-define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -146,7 +146,7 @@ define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %o
ret void
}
-define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_log_bf16_div2_imm_first:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -175,7 +175,7 @@ define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %o
}
-define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_rcp_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -203,7 +203,7 @@ define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_rsq_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -231,7 +231,7 @@ define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_sin_bf16_div2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -259,7 +259,7 @@ define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_sqrt_bf16_mul2:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -287,7 +287,7 @@ define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) {
ret void
}
-define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
+define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_tanh_bf16_mul4:
; SDAG-FAKE16: ; %bb.0:
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -314,3 +314,5 @@ define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) {
store bfloat %mul4, ptr addrspace(1) %out
ret void
}
+
+attributes #0 = { nounwind denormal_fpenv(preservesign) }
>From a2597ecc07633b2796a53ee73428636c7c91e627 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Fri, 28 Aug 2026 12:02:04 -0700
Subject: [PATCH 3/3] [AMDGPU] Use FP64FP16Denormals to check denormal flushing
bf16 has no denormal mode of its own; it follows the default
("denormal-fp-math") mode, which is the same field as f64/f16.
Add a few tests to reflect the effect of denormal attributes
on omod folding.
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 11 ++-
llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll | 100 ++++++++++++++++++++
2 files changed, 107 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 1f0750d66684c..0cf4e826ad924 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2504,8 +2504,10 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
return {nullptr, SIOutMods::NONE};
}
case AMDGPU::V_PK_MUL_BF16: {
- // OMOD folding for BF16 packed multiply
- if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign ||
+ // OMOD folding for BF16 packed multiply. bf16 has no denormal mode of its
+ // own; it follows the default ("denormal-fp-math") mode, which is the same
+ // field as f64/f16.
+ if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign ||
MI.mayRaiseFPException())
return {nullptr, SIOutMods::NONE};
@@ -2536,8 +2538,9 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
return {Src0, OMod};
}
case AMDGPU::V_PK_ADD_BF16: {
- // OMOD folding for BF16 packed add: x + x -> x * 2
- if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign)
+ // OMOD folding for BF16 packed add: x + x -> x * 2. See the bf16 denormal
+ // mode note in the V_PK_MUL_BF16 case above.
+ if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)
return {nullptr, SIOutMods::NONE};
const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
index dba56ca84f0a1..02a8e925db226 100644
--- a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -315,4 +315,104 @@ define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
ret void
}
+; Output denormals enabled for every type: no omod folding.
+define amdgpu_ps void @v_cos_bf16_mul2_denorm_ieee(bfloat %in, ptr addrspace(1) %out) #1 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_denorm_ieee:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e32 v0, v0
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2_denorm_ieee:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-REAL16-NEXT: v_pk_add_bf16 v0, v0, v0
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %cos, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+; Only f32 output denormals are flushed. bf16 follows the default mode, which
+; is IEEE here, so there is still no omod folding.
+define amdgpu_ps void @v_cos_bf16_mul2_f32_denorm_flush(bfloat %in, ptr addrspace(1) %out) #2 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_f32_denorm_flush:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e32 v0, v0
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2_f32_denorm_flush:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-REAL16-NEXT: v_pk_add_bf16 v0, v0, v0
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %cos, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+; The default mode flushes output denormals, so bf16 does too, even though f32
+; is IEEE: omod folding applies.
+define amdgpu_ps void @v_cos_bf16_mul2_default_denorm_flush(bfloat %in, ptr addrspace(1) %out) #3 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_default_denorm_flush:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2_default_denorm_flush:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %cos, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
attributes #0 = { nounwind denormal_fpenv(preservesign) }
+attributes #1 = { nounwind denormal_fpenv(ieee) }
+attributes #2 = { nounwind denormal_fpenv(ieee, float: preservesign) }
+attributes #3 = { nounwind denormal_fpenv(preservesign, float: ieee) }
More information about the llvm-commits
mailing list