[llvm] [AMDGPU] Add OMOD folding for TRANS bfloat16 instructions (PR #218286)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 13:51:07 PDT 2026
https://github.com/changpeng created https://github.com/llvm/llvm-project/pull/218286
Enable omod folding optimization for transcendental bfloat16 instructions. This patch extends SIFoldOperands to recognize and fold multiplication or division patterns following TRANS BF16 instructions into the OMOD field, allowing operations like `rsq.bf16(x) * 4.0` to be encoded as `rsq.bf16(x) mul:4`.
Key implementation details:
1. Added V_PK_MUL_BF16 support in isOMod() and getOModValue():
- Detects multiplication by 2.0 (0x4000), 4.0 (0x4080), or 0.5 (0x3F00)
- Only folds when the immediate is in src1 operand
- Allows op_sel_hi modifier but blocks other modifiers (NEG, ABS)
2. Added V_PK_ADD_BF16 support for x+x pattern:
- Recognizes when both operands are the same register
- Folds to MUL2 OMOD
3. Enhanced tryFoldOMod() for real-true16 mode:
- Handles REG_SEQUENCE that packs vgpr_16 results into vgpr_32
- Validates high 16 bits are IMPLICIT_DEF before looking through
- Preserves REG_SEQUENCE output register for replacement to avoid register class mismatches
The optimization requires denormal mode preservesign and no FP exceptions. Works correctly in both fake16 and real-true16 modes.
>From ab68253fdd2915b8b2d4b920557f1f5672a765c4 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Sun, 23 Aug 2026 13:44:29 -0700
Subject: [PATCH] [AMDGPU] Add OMOD folding for TRANS bfloat16 instructions
Enable omod folding optimization for transcendental bfloat16 instructions.
This patch extends SIFoldOperands to recognize and fold multiplication or
division patterns following TRANS BF16 instructions into the OMOD field,
allowing operations like `rsq.bf16(x) * 4.0` to be encoded as `rsq.bf16(x) mul:4`.
Key implementation details:
1. Added V_PK_MUL_BF16 support in isOMod() and getOModValue():
- Detects multiplication by 2.0 (0x4000), 4.0 (0x4080), or 0.5 (0x3F00)
- Only folds when the immediate is in src1 operand
- Allows op_sel_hi modifier but blocks other modifiers (NEG, ABS)
2. Added V_PK_ADD_BF16 support for x+x pattern:
- Recognizes when both operands are the same register
- Folds to MUL2 OMOD
3. Enhanced tryFoldOMod() for real-true16 mode:
- Handles REG_SEQUENCE that packs vgpr_16 results into vgpr_32
- Validates high 16 bits are IMPLICIT_DEF before looking through
- Preserves REG_SEQUENCE output register for replacement to avoid
register class mismatches
The optimization requires denormal mode preservesign and no FP exceptions.
Works correctly in both fake16 and real-true16 modes.
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 95 +++++-
llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll | 318 ++++++++++++++++++++
2 files changed, 412 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 584efa001f2a5..9d1fc8deb9356 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2414,6 +2414,18 @@ static int getOModValue(unsigned Opc, int64_t Val) {
return SIOutMods::NONE;
}
}
+ case AMDGPU::V_PK_MUL_BF16: {
+ switch (static_cast<uint16_t>(Val)) {
+ case 0x3F00: // 0.5 in BF16
+ return SIOutMods::DIV2;
+ case 0x4000: // 2.0 in BF16
+ return SIOutMods::MUL2;
+ case 0x4080: // 4.0 in BF16
+ return SIOutMods::MUL4;
+ default:
+ return SIOutMods::NONE;
+ }
+ }
default:
llvm_unreachable("invalid mul opcode");
}
@@ -2491,6 +2503,63 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
return {nullptr, SIOutMods::NONE};
}
+ case AMDGPU::V_PK_MUL_BF16: {
+ // OMOD folding for BF16 packed multiply
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign ||
+ MI.mayRaiseFPException())
+ return {nullptr, SIOutMods::NONE};
+
+ const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+
+ // If there is an immediate operand, it must be Src1
+ std::optional<int64_t> Src1Imm =
+ TII->getImmOrMaterializedImm(const_cast<MachineOperand &>(*Src1));
+ if (!Src1Imm)
+ return {nullptr, SIOutMods::NONE};
+
+ int OMod = getOModValue(AMDGPU::V_PK_MUL_BF16, *Src1Imm);
+ if (OMod == SIOutMods::NONE)
+ return {nullptr, SIOutMods::NONE};
+
+ // Modifiers other than op_sel_hi block OMOD folding
+ const MachineOperand *Src0Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ const MachineOperand *Src1Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
+ return {nullptr, SIOutMods::NONE};
+
+ return {Src0, OMod};
+ }
+ case AMDGPU::V_PK_ADD_BF16: {
+ // OMOD folding for BF16 packed add: x + x -> x * 2
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign)
+ return {nullptr, SIOutMods::NONE};
+
+ const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+
+ if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
+ Src0->getSubReg() != Src1->getSubReg())
+ return {nullptr, SIOutMods::NONE};
+
+ // Modifiers other than op_sel_hi block OMOD folding
+ const MachineOperand *Src0Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ const MachineOperand *Src1Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
+ return {nullptr, SIOutMods::NONE};
+
+ return {Src0, SIOutMods::MUL2};
+ }
default:
return {nullptr, SIOutMods::NONE};
}
@@ -2507,6 +2576,31 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
return false;
MachineInstr *Def = MRI->getVRegDef(RegOp->getReg());
+ Register OModSrcReg = Def->getOperand(0).getReg();
+
+ // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
+ // REG_SEQUENCE. Look through it to find the actual instruction.
+ if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
+ Def->getOperand(1).isReg() && Def->getOperand(2).isImm() &&
+ Def->getOperand(2).getImm() == AMDGPU::lo16 &&
+ Def->getOperand(3).isReg()) {
+ // Only look through if the high 16 bits are undefined
+ bool CanLookThrough = true;
+ MachineInstr *Hi16Def = MRI->getVRegDef(Def->getOperand(3).getReg());
+ if (!Hi16Def || !Hi16Def->isImplicitDef())
+ CanLookThrough = false;
+
+ if (CanLookThrough) {
+ Register SrcReg = Def->getOperand(1).getReg();
+ if (!MRI->hasOneNonDBGUse(SrcReg))
+ return false;
+
+ Def = MRI->getVRegDef(SrcReg);
+ if (!Def)
+ return false;
+ }
+ }
+
MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod);
if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
return false;
@@ -2522,7 +2616,6 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
DefOMod->setImm(OMod);
- Register OModSrcReg = Def->getOperand(0).getReg();
MRI->replaceRegWith(MI.getOperand(0).getReg(), OModSrcReg);
// Kill flags can be wrong if we replaced a def inside a loop with a def
// outside the loop.
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
new file mode 100644
index 0000000000000..ec82d9ea90aaf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s
+; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=GISEL-FAKE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-REAL16 %s
+; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=GISEL-REAL16 %s
+
+; FIXME: GlobalISel does not work with bf16
+
+define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %cos, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_exp_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_exp_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_exp_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat %exp, 4.0
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_log_bf16_div2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_log_bf16_div2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_log_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat %log, 0.5
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat 2.0, %cos
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_exp_bf16_mul4_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_exp_bf16_mul4_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_exp_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat 4.0, %exp
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_log_bf16_div2_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_log_bf16_div2_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_log_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat 0.5, %log
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+
+define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_rcp_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_rcp_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_rcp_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_rcp_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %rcp, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_rsq_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_rsq_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_rsq_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_rsq_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat %rsq, 4.0
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_sin_bf16_div2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_sin_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_sin_bf16_div2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_sin_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat %sin, 0.5
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_sqrt_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_sqrt_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_sqrt_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_sqrt_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %sqrt = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %sqrt, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_tanh_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_tanh_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_tanh_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_tanh_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %tanh = call bfloat @llvm.amdgcn.tanh.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat %tanh, 4.0
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { nounwind denormal_fpenv(preservesign) }
More information about the llvm-commits
mailing list