[llvm] [AMDGPU] Add OMOD folding for TRANS bfloat16 instructions (PR #218286)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 13:51:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Changpeng Fang (changpeng)
<details>
<summary>Changes</summary>
Enable omod folding optimization for transcendental bfloat16 instructions. This patch extends SIFoldOperands to recognize and fold multiplication or division patterns following TRANS BF16 instructions into the OMOD field, allowing operations like `rsq.bf16(x) * 4.0` to be encoded as `rsq.bf16(x) mul:4`.
Key implementation details:
1. Added V_PK_MUL_BF16 support in isOMod() and getOModValue():
- Detects multiplication by 2.0 (0x4000), 4.0 (0x4080), or 0.5 (0x3F00)
- Only folds when the immediate is in src1 operand
- Allows op_sel_hi modifier but blocks other modifiers (NEG, ABS)
2. Added V_PK_ADD_BF16 support for x+x pattern:
- Recognizes when both operands are the same register
- Folds to MUL2 OMOD
3. Enhanced tryFoldOMod() for real-true16 mode:
- Handles REG_SEQUENCE that packs vgpr_16 results into vgpr_32
- Validates high 16 bits are IMPLICIT_DEF before looking through
- Preserves REG_SEQUENCE output register for replacement to avoid register class mismatches
The optimization requires denormal mode preservesign and no FP exceptions. Works correctly in both fake16 and real-true16 modes.
---
Patch is 20.23 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218286.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+94-1)
- (added) llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll (+318)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 584efa001f2a5..9d1fc8deb9356 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2414,6 +2414,18 @@ static int getOModValue(unsigned Opc, int64_t Val) {
return SIOutMods::NONE;
}
}
+ case AMDGPU::V_PK_MUL_BF16: {
+ switch (static_cast<uint16_t>(Val)) {
+ case 0x3F00: // 0.5 in BF16
+ return SIOutMods::DIV2;
+ case 0x4000: // 2.0 in BF16
+ return SIOutMods::MUL2;
+ case 0x4080: // 4.0 in BF16
+ return SIOutMods::MUL4;
+ default:
+ return SIOutMods::NONE;
+ }
+ }
default:
llvm_unreachable("invalid mul opcode");
}
@@ -2491,6 +2503,63 @@ SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
return {nullptr, SIOutMods::NONE};
}
+ case AMDGPU::V_PK_MUL_BF16: {
+ // OMOD folding for BF16 packed multiply
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign ||
+ MI.mayRaiseFPException())
+ return {nullptr, SIOutMods::NONE};
+
+ const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+
+ // If there is an immediate operand, it must be Src1
+ std::optional<int64_t> Src1Imm =
+ TII->getImmOrMaterializedImm(const_cast<MachineOperand &>(*Src1));
+ if (!Src1Imm)
+ return {nullptr, SIOutMods::NONE};
+
+ int OMod = getOModValue(AMDGPU::V_PK_MUL_BF16, *Src1Imm);
+ if (OMod == SIOutMods::NONE)
+ return {nullptr, SIOutMods::NONE};
+
+ // Modifiers other than op_sel_hi block OMOD folding
+ const MachineOperand *Src0Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ const MachineOperand *Src1Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
+ return {nullptr, SIOutMods::NONE};
+
+ return {Src0, OMod};
+ }
+ case AMDGPU::V_PK_ADD_BF16: {
+ // OMOD folding for BF16 packed add: x + x -> x * 2
+ if (MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign)
+ return {nullptr, SIOutMods::NONE};
+
+ const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+
+ if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
+ Src0->getSubReg() != Src1->getSubReg())
+ return {nullptr, SIOutMods::NONE};
+
+ // Modifiers other than op_sel_hi block OMOD folding
+ const MachineOperand *Src0Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
+ const MachineOperand *Src1Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
+ if ((Src0Mods && (Src0Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ (Src1Mods && (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1)) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
+ TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
+ return {nullptr, SIOutMods::NONE};
+
+ return {Src0, SIOutMods::MUL2};
+ }
default:
return {nullptr, SIOutMods::NONE};
}
@@ -2507,6 +2576,31 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
return false;
MachineInstr *Def = MRI->getVRegDef(RegOp->getReg());
+ Register OModSrcReg = Def->getOperand(0).getReg();
+
+ // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
+ // REG_SEQUENCE. Look through it to find the actual instruction.
+ if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
+ Def->getOperand(1).isReg() && Def->getOperand(2).isImm() &&
+ Def->getOperand(2).getImm() == AMDGPU::lo16 &&
+ Def->getOperand(3).isReg()) {
+ // Only look through if the high 16 bits are undefined
+ bool CanLookThrough = true;
+ MachineInstr *Hi16Def = MRI->getVRegDef(Def->getOperand(3).getReg());
+ if (!Hi16Def || !Hi16Def->isImplicitDef())
+ CanLookThrough = false;
+
+ if (CanLookThrough) {
+ Register SrcReg = Def->getOperand(1).getReg();
+ if (!MRI->hasOneNonDBGUse(SrcReg))
+ return false;
+
+ Def = MRI->getVRegDef(SrcReg);
+ if (!Def)
+ return false;
+ }
+ }
+
MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod);
if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
return false;
@@ -2522,7 +2616,6 @@ bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
DefOMod->setImm(OMod);
- Register OModSrcReg = Def->getOperand(0).getReg();
MRI->replaceRegWith(MI.getOperand(0).getReg(), OModSrcReg);
// Kill flags can be wrong if we replaced a def inside a loop with a def
// outside the loop.
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
new file mode 100644
index 0000000000000..ec82d9ea90aaf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s
+; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=GISEL-FAKE16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-REAL16 %s
+; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=GISEL-REAL16 %s
+
+; FIXME: GlobalISel does not work with bf16
+
+define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %cos, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_exp_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_exp_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_exp_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat %exp, 4.0
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_log_bf16_div2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_log_bf16_div2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_log_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat %log, 0.5
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_cos_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_cos_bf16_mul2_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_cos_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat 2.0, %cos
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_exp_bf16_mul4_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_exp_bf16_mul4_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_exp_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat 4.0, %exp
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_log_bf16_div2_imm_first:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_log_bf16_div2_imm_first:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_log_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat 0.5, %log
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+
+define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_rcp_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_rcp_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_rcp_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_rcp_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %rcp, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_rsq_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_rsq_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_rsq_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_rsq_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %in)
+ %mul4 = fmul nsz bfloat %rsq, 4.0
+ store bfloat %mul4, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_sin_bf16_div2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_sin_bf16_e64 v0, v0 div:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_sin_bf16_div2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_sin_bf16_e64 v0.l, v0.l div:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat %in)
+ %div2 = fmul nsz bfloat %sin, 0.5
+ store bfloat %div2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_sqrt_bf16_mul2:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_sqrt_bf16_e64 v0, v0 mul:2
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_sqrt_bf16_mul2:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_sqrt_bf16_e64 v0.l, v0.l mul:2
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: s_endpgm
+ %sqrt = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat %in)
+ %mul2 = fmul nsz bfloat %sqrt, 2.0
+ store bfloat %mul2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
+; SDAG-FAKE16-LABEL: v_tanh_bf16_mul4:
+; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-FAKE16-NEXT: v_nop
+; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-FAKE16-NEXT: v_tanh_bf16_e64 v0, v0 mul:4
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: s_endpgm
+;
+; SDAG-REAL16-LABEL: v_tanh_bf16_mul4:
+; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; SDAG-REAL16-NEXT: v_nop
+; SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; SDAG-REAL16-NEXT: v_tanh_bf16_e64 v0.l, v0.l mul:4
+; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218286
More information about the llvm-commits
mailing list