[llvm] AMDGPU: Add tonearest and towardzero roundings for intrinsic llvm.fptrunc.round (PR #104486)
Changpeng Fang via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 16 11:31:04 PDT 2024
https://github.com/changpeng updated https://github.com/llvm/llvm-project/pull/104486
>From 621d6b988124140164cbcdb0a9cb38d9d3ff9bf7 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Thu, 15 Aug 2024 11:01:44 -0700
Subject: [PATCH 1/3] AMDGPU: Improve codegen for intrinsic llvm.fptrunc.round
This work simplifies and generalizes the intruction definition and
code generation for intrinsic llvm.fptrunc.round. We no longer name
the instruction with the rounding mode. Instead, we introduce an
immediate operand for the rounding mode for the pseudo instruction.
This immediate will be used to set up the hardware mode register at
the time the real instruction is generated. We name the pseudo instruction
as FPTRUNC_ROUND_F32_TO_F16 (for f32 -> f16), which is easy to generalize for
other types.
"round.towardzero" and "round.tonearest" are added for f32 -> f16
truncating, in addition to the existing "round.upward" and "round.downward".
Other rounding modes are not supported by hardware at this moment.
We can easily extend this work to support f64 -> f32 and f64 -> f16,
which will be the patch that follows. Support of truncating to bfloat
is also in the plan.
---
llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 3 +-
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 3 +-
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 3 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 28 ++++++----
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 3 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 42 ++++++++-------
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 10 ++--
llvm/lib/Target/AMDGPU/SIInstructions.td | 28 +++-------
llvm/lib/Target/AMDGPU/SIModeRegister.cpp | 29 +++--------
.../CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll | 9 ++--
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 19 +++++++
.../CodeGen/AMDGPU/mode-register-fptrunc.mir | 52 ++++++++++++++++++-
14 files changed, 140 insertions(+), 93 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 825c1de6c91f12..8bee84b8a87f27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -297,8 +297,7 @@ def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_UBYTE, SIsbuffer_load_ubyte>;
def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_SSHORT, SIsbuffer_load_short>;
def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_USHORT, SIsbuffer_load_ushort>;
-def : GINodeEquiv<G_FPTRUNC_ROUND_UPWARD, SIfptrunc_round_upward>;
-def : GINodeEquiv<G_FPTRUNC_ROUND_DOWNWARD, SIfptrunc_round_downward>;
+def : GINodeEquiv<G_FPTRUNC_ROUND, SIfptrunc_round>;
class GISelSop2Pat <
SDPatternOperator node,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 8c8950957dd819..e57c8f8b7b4835 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5500,8 +5500,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
NODE_NAME_CASE(CONST_DATA_PTR)
NODE_NAME_CASE(PC_ADD_REL_OFFSET)
NODE_NAME_CASE(LDS)
- NODE_NAME_CASE(FPTRUNC_ROUND_UPWARD)
- NODE_NAME_CASE(FPTRUNC_ROUND_DOWNWARD)
+ NODE_NAME_CASE(FPTRUNC_ROUND)
NODE_NAME_CASE(DUMMY_CHAIN)
case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
NODE_NAME_CASE(LOAD_D16_HI)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 1a5244f7ec809b..59f640ea99de3e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -553,8 +553,7 @@ enum NodeType : unsigned {
CONST_DATA_PTR,
PC_ADD_REL_OFFSET,
LDS,
- FPTRUNC_ROUND_UPWARD,
- FPTRUNC_ROUND_DOWNWARD,
+ FPTRUNC_ROUND,
DUMMY_CHAIN,
FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c6c4b8f9306471..8ea1a30a340052 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2180,7 +2180,7 @@ bool AMDGPULegalizerInfo::legalizeCustom(
case TargetOpcode::G_CTLZ_ZERO_UNDEF:
return legalizeCTLZ_ZERO_UNDEF(MI, MRI, B);
case TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND:
- return legalizeFPTruncRound(MI, B);
+ return legalizeFPTruncRound(MI, MRI, B);
case TargetOpcode::G_STACKSAVE:
return legalizeStackSave(MI, B);
case TargetOpcode::G_GET_FPENV:
@@ -7089,23 +7089,29 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
}
bool AMDGPULegalizerInfo::legalizeFPTruncRound(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
- unsigned Opc;
- int RoundMode = MI.getOperand(2).getImm();
+ Register Src = MI.getOperand(1).getReg();
+ if (MRI.getType(Src) != LLT::scalar(32))
+ return false;
- if (RoundMode == (int)RoundingMode::TowardPositive)
- Opc = AMDGPU::G_FPTRUNC_ROUND_UPWARD;
- else if (RoundMode == (int)RoundingMode::TowardNegative)
- Opc = AMDGPU::G_FPTRUNC_ROUND_DOWNWARD;
- else
+ // Only support towardzero, tonearest, upward and downward.
+ int RoundMode = MI.getOperand(2).getImm();
+ if (RoundMode < (int)RoundingMode::TowardZero ||
+ RoundMode > (int)RoundingMode::TowardNegative)
return false;
- B.buildInstr(Opc)
+ // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
+ // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+ // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
+ // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
+ unsigned HW_Mode = (RoundMode + 3) % 4;
+ B.buildInstr(AMDGPU::G_FPTRUNC_ROUND)
.addDef(MI.getOperand(0).getReg())
- .addUse(MI.getOperand(1).getReg());
+ .addUse(Src)
+ .addImm(HW_Mode);
MI.eraseFromParent();
-
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index db1c5874093a71..4dd62800e7aee5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -212,7 +212,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
bool legalizeBVHIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const;
- bool legalizeFPTruncRound(MachineInstr &MI, MachineIRBuilder &B) const;
+ bool legalizeFPTruncRound(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) const;
bool legalizeStackSave(MachineInstr &MI, MachineIRBuilder &B) const;
bool legalizeWaveID(MachineInstr &MI, MachineIRBuilder &B) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 17067ddd93ff08..5add368c05646a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -5259,8 +5259,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
break;
}
- case AMDGPU::G_FPTRUNC_ROUND_UPWARD:
- case AMDGPU::G_FPTRUNC_ROUND_DOWNWARD:
+ case AMDGPU::G_FPTRUNC_ROUND:
return getDefaultMappingVOP(MI);
case AMDGPU::G_PREFETCH:
OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 25fee559faa29c..01651c06b00323 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5804,24 +5804,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::FP_ROUND:
case ISD::STRICT_FP_ROUND:
return lowerFP_ROUND(Op, DAG);
- case ISD::FPTRUNC_ROUND: {
- unsigned Opc;
- SDLoc DL(Op);
-
- if (Op.getOperand(0)->getValueType(0) != MVT::f32)
- return SDValue();
-
- // Get the rounding mode from the last operand
- int RoundMode = Op.getConstantOperandVal(1);
- if (RoundMode == (int)RoundingMode::TowardPositive)
- Opc = AMDGPUISD::FPTRUNC_ROUND_UPWARD;
- else if (RoundMode == (int)RoundingMode::TowardNegative)
- Opc = AMDGPUISD::FPTRUNC_ROUND_DOWNWARD;
- else
- return SDValue();
-
- return DAG.getNode(Opc, DL, Op.getNode()->getVTList(), Op->getOperand(0));
- }
+ case ISD::FPTRUNC_ROUND:
+ return lowerFPTRUNC_ROUND(Op, DAG);
case ISD::TRAP:
return lowerTRAP(Op, DAG);
case ISD::DEBUGTRAP:
@@ -6671,6 +6655,28 @@ SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
DAG.getTargetConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerFPTRUNC_ROUND(SDValue Op,
+ SelectionDAG &DAG) const {
+ if (Op.getOperand(0)->getValueType(0) != MVT::f32)
+ return SDValue();
+
+ // Only support towardzero, tonearest, upward and downward.
+ int RoundMode = Op.getConstantOperandVal(1);
+ if (RoundMode < (int)RoundingMode::TowardZero ||
+ RoundMode > (int)RoundingMode::TowardNegative)
+ return SDValue();
+
+ // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
+ // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+ // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
+ // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
+ unsigned HW_Mode = (RoundMode + 3) % 4;
+ SDLoc DL(Op);
+ unsigned Opc = AMDGPUISD::FPTRUNC_ROUND;
+ return DAG.getNode(Opc, DL, Op.getNode()->getVTList(), Op->getOperand(0),
+ DAG.getTargetConstant(HW_Mode, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
assert(Op.getValueType() == MVT::f16 &&
"Do not know how to custom lower FP_ROUND for non-f16 type");
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 1f198a92c0fa6a..eed4b3e79cdeee 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -145,6 +145,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFPTRUNC_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerMUL(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index e99b43afd1c3a2..85281713e22b1f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -304,13 +304,11 @@ def SIdenorm_mode : SDNode<"AMDGPUISD::DENORM_MODE",
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]
>;
-def SIfptrunc_round_upward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_UPWARD",
- SDTFPRoundOp
->;
+def SDTFPRoundModeOp : SDTypeProfile<1, 2, [
+ SDTCisFP<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
+]>;
-def SIfptrunc_round_downward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_DOWNWARD",
- SDTFPRoundOp
->;
+def SIfptrunc_round : SDNode<"AMDGPUISD::FPTRUNC_ROUND", SDTFPRoundModeOp>;
//===----------------------------------------------------------------------===//
// ValueType helpers
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 5a139d1cf8d825..0b911004e098f4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -225,20 +225,12 @@ def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
>;
} // End usesCustomInserter = 1
-// Pseudo instructions used for @llvm.fptrunc.round upward
-// and @llvm.fptrunc.round downward.
-// These intrinsics will be legalized to G_FPTRUNC_ROUND_UPWARD
-// and G_FPTRUNC_ROUND_DOWNWARD before being lowered to
-// FPTRUNC_UPWARD_PSEUDO and FPTRUNC_DOWNWARD_PSEUDO.
-// The final codegen is done in the ModeRegister pass.
+// Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done
+// in the ModeRegister pass.
let Uses = [MODE, EXEC] in {
-def FPTRUNC_UPWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
- (ins VGPR_32:$src0),
- [(set f16:$vdst, (SIfptrunc_round_upward f32:$src0))]>;
-
-def FPTRUNC_DOWNWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
- (ins VGPR_32:$src0),
- [(set f16:$vdst, (SIfptrunc_round_downward f32:$src0))]>;
+def FPTRUNC_ROUND_F32_TO_F16 : VPseudoInstSI <(outs VGPR_32:$vdst),
+ (ins VGPR_32:$src0, i32imm:$round),
+ [(set f16:$vdst, (SIfptrunc_round f32:$src0, i32:$round))]>;
} // End Uses = [MODE, EXEC]
// Invert the exec mask and overwrite the inactive lanes of dst with inactive,
@@ -4072,15 +4064,9 @@ def G_SI_CALL : AMDGPUGenericInstruction {
let isConvergent = 1;
}
-def G_FPTRUNC_ROUND_UPWARD : AMDGPUGenericInstruction {
- let OutOperandList = (outs type0:$vdst);
- let InOperandList = (ins type1:$src0);
- let hasSideEffects = 0;
-}
-
-def G_FPTRUNC_ROUND_DOWNWARD : AMDGPUGenericInstruction {
+def G_FPTRUNC_ROUND : AMDGPUGenericInstruction {
let OutOperandList = (outs type0:$vdst);
- let InOperandList = (ins type1:$src0);
+ let InOperandList = (ins type1:$src0, untyped_imm_0:$round);
let hasSideEffects = 0;
}
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index e7f448233ca347..a7bfa2216edf68 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -163,17 +163,19 @@ FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); }
// double precision setting.
Status SIModeRegister::getInstructionMode(MachineInstr &MI,
const SIInstrInfo *TII) {
+ unsigned Opcode = MI.getOpcode();
if (TII->usesFPDPRounding(MI) ||
- MI.getOpcode() == AMDGPU::FPTRUNC_UPWARD_PSEUDO ||
- MI.getOpcode() == AMDGPU::FPTRUNC_DOWNWARD_PSEUDO) {
- switch (MI.getOpcode()) {
+ Opcode == AMDGPU::FPTRUNC_ROUND_F32_TO_F16) {
+ switch (Opcode) {
case AMDGPU::V_INTERP_P1LL_F16:
case AMDGPU::V_INTERP_P1LV_F16:
case AMDGPU::V_INTERP_P2_F16:
// f16 interpolation instructions need double precision round to zero
return Status(FP_ROUND_MODE_DP(3),
FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
- case AMDGPU::FPTRUNC_UPWARD_PSEUDO: {
+ case AMDGPU::FPTRUNC_ROUND_F32_TO_F16: {
+ unsigned Mode = MI.getOperand(2).getImm();
+ MI.removeOperand(2);
// Replacing the pseudo by a real instruction in place
if (TII->getSubtarget().hasTrue16BitInsts()) {
MachineBasicBlock &MBB = *MI.getParent();
@@ -188,24 +190,7 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
} else
MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
return Status(FP_ROUND_MODE_DP(3),
- FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_INF));
- }
- case AMDGPU::FPTRUNC_DOWNWARD_PSEUDO: {
- // Replacing the pseudo by a real instruction in place
- if (TII->getSubtarget().hasTrue16BitInsts()) {
- MachineBasicBlock &MBB = *MI.getParent();
- MachineInstrBuilder B(*MBB.getParent(), MI);
- MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
- MachineOperand Src0 = MI.getOperand(1);
- MI.removeOperand(1);
- B.addImm(0); // src0_modifiers
- B.add(Src0); // re-add src0 operand
- B.addImm(0); // clamp
- B.addImm(0); // omod
- } else
- MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
- return Status(FP_ROUND_MODE_DP(3),
- FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEGINF));
+ FP_ROUND_MODE_DP(Mode));
}
default:
return DefaultStatus;
diff --git a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
index f312f3138c08d3..9fa3eb22a554a8 100644
--- a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
@@ -1,8 +1,9 @@
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefixes=SDAG-FAIL
+; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=GISEL-FAIL
-define amdgpu_gs void @test_fptrunc_round_legalization(double %a, i32 %data0, <4 x i32> %data1, ptr addrspace(1) %out) {
-; FAIL: LLVM ERROR: Cannot select
+define amdgpu_gs void @test_fptrunc_round_f64(double %a, ptr addrspace(1) %out) {
+; SDAG-FAIL: LLVM ERROR: Cannot select
+; GISEL-FAIL: unable to legalize instruction
%res = call half @llvm.fptrunc.round.f64(double %a, metadata !"round.upward")
store half %res, ptr addrspace(1) %out, align 4
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index b8c16d2ed3b2f1..b32f719e2d27a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -3,6 +3,15 @@
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_tonearest(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_tonearest:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT: ; return to shader part epilog
+ %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.tonearest")
+ ret half %res
+}
+
define amdgpu_gs half @v_fptrunc_round_f32_to_f16_upward(float %a) {
; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward:
; CHECK: ; %bb.0:
@@ -23,6 +32,16 @@ define amdgpu_gs half @v_fptrunc_round_f32_to_f16_downward(float %a) {
ret half %res
}
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_towardzero(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_towardzero:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; CHECK-NEXT: v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT: ; return to shader part epilog
+ %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.towardzero")
+ ret half %res
+}
+
define amdgpu_gs void @v_fptrunc_round_f32_to_f16_upward_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward_multiple_calls:
; CHECK: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
index fda33f52bf06d2..78b29132d90186 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
@@ -3,6 +3,29 @@
# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass si-mode-register %s -o - | FileCheck %s --check-prefixes=CHECK
# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass si-mode-register %s -o - | FileCheck %s --check-prefixes=GFX11
+---
+name: ftrunc_tonearest
+
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: ftrunc_tonearest
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-LABEL: name: ftrunc_tonearest
+ ; GFX11: liveins: $sgpr0
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ $vgpr1 = FPTRUNC_ROUND_F32_TO_F16 $vgpr0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
---
name: ftrunc_upward
@@ -25,7 +48,7 @@ body: |
; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
; GFX11-NEXT: S_ENDPGM 0
$vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- $vgpr1 = FPTRUNC_UPWARD_PSEUDO $vgpr0, implicit $mode, implicit $exec
+ $vgpr1 = FPTRUNC_ROUND_F32_TO_F16 $vgpr0, 1, implicit $mode, implicit $exec
S_ENDPGM 0
...
---
@@ -50,6 +73,31 @@ body: |
; GFX11-NEXT: $vgpr0 = V_CVT_F16_F32_t16_e64 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
; GFX11-NEXT: S_ENDPGM 0
$vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- $vgpr0 = FPTRUNC_DOWNWARD_PSEUDO $vgpr1, implicit $mode, implicit $exec
+ $vgpr0 = FPTRUNC_ROUND_F32_TO_F16 $vgpr1, 2, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+---
+name: ftrunc_towardzero
+
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: ftrunc_towardzero
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-LABEL: name: ftrunc_towardzero
+ ; GFX11: liveins: $sgpr0
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+ ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ $vgpr1 = FPTRUNC_ROUND_F32_TO_F16 $vgpr0, 3, implicit $mode, implicit $exec
S_ENDPGM 0
...
>From 3fd140645b16999f11768c928cf21a2d8662b314 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Fri, 16 Aug 2024 10:22:30 -0700
Subject: [PATCH 2/3] AMDGPU: Add tonearest and towardzero support for
llvm.fptrunc.round
This work simplifies and generalizes the instruction definition and
for intrinsic llvm.fptrunc.round. We no longer name the instruction with
the rounding mode. Instead, we introduce an immediate operand for the
rounding mode for the pseudo instruction. This immediate will be used
to set up the hardware mode register at the time the real instruction is
generated. We name the pseudo instruction as FPTRUNC_ROUND_F16_F32
(for f32 -> f16), which is easy to generalize for other types.
"round.towardzero" and "round.tonearest" are added for f32 -> f16
truncating, in addition to the existing "round.upward" and "round.downward".
Other rounding modes are not supported by hardware at this moment.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +++-
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 +-
llvm/lib/Target/AMDGPU/SIModeRegister.cpp | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 01651c06b00323..5316bd077f55f7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6660,7 +6660,7 @@ SDValue SITargetLowering::lowerFPTRUNC_ROUND(SDValue Op,
if (Op.getOperand(0)->getValueType(0) != MVT::f32)
return SDValue();
- // Only support towardzero, tonearest, upward and downward.
+ // Only support towardzero, tonearest, upward and downward.
int RoundMode = Op.getConstantOperandVal(1);
if (RoundMode < (int)RoundingMode::TowardZero ||
RoundMode > (int)RoundingMode::TowardNegative)
@@ -6681,6 +6681,8 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
assert(Op.getValueType() == MVT::f16 &&
"Do not know how to custom lower FP_ROUND for non-f16 type");
+ return Op;
+
SDValue Src = Op.getOperand(0);
EVT SrcVT = Src.getValueType();
if (SrcVT != MVT::f64)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 0b911004e098f4..3ed2dc1236ba1c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -228,7 +228,7 @@ def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
// Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done
// in the ModeRegister pass.
let Uses = [MODE, EXEC] in {
-def FPTRUNC_ROUND_F32_TO_F16 : VPseudoInstSI <(outs VGPR_32:$vdst),
+def FPTRUNC_ROUND_F16_F32 : VPseudoInstSI <(outs VGPR_32:$vdst),
(ins VGPR_32:$src0, i32imm:$round),
[(set f16:$vdst, (SIfptrunc_round f32:$src0, i32:$round))]>;
} // End Uses = [MODE, EXEC]
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index a7bfa2216edf68..3758f522daac8b 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -165,7 +165,7 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
const SIInstrInfo *TII) {
unsigned Opcode = MI.getOpcode();
if (TII->usesFPDPRounding(MI) ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F32_TO_F16) {
+ Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32) {
switch (Opcode) {
case AMDGPU::V_INTERP_P1LL_F16:
case AMDGPU::V_INTERP_P1LV_F16:
@@ -173,7 +173,7 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
// f16 interpolation instructions need double precision round to zero
return Status(FP_ROUND_MODE_DP(3),
FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
- case AMDGPU::FPTRUNC_ROUND_F32_TO_F16: {
+ case AMDGPU::FPTRUNC_ROUND_F16_F32: {
unsigned Mode = MI.getOperand(2).getImm();
MI.removeOperand(2);
// Replacing the pseudo by a real instruction in place
>From 0a7d6c9b8d4bc2ae9933b895b138b4f3d410490c Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang at amd.com>
Date: Fri, 16 Aug 2024 11:27:32 -0700
Subject: [PATCH 3/3] AMDGPU: Add tonearest and towardzero roundings for
intrinsic llvm.fptrunc.round
Update based on comments:
1. Do exact equality checks for the supported modes.
2. Remove a bad leftover code in LowerFP_ROUND.
---
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 6 ++++--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 8 ++++----
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 8ea1a30a340052..cf7269962236a0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -7097,8 +7097,10 @@ bool AMDGPULegalizerInfo::legalizeFPTruncRound(MachineInstr &MI,
// Only support towardzero, tonearest, upward and downward.
int RoundMode = MI.getOperand(2).getImm();
- if (RoundMode < (int)RoundingMode::TowardZero ||
- RoundMode > (int)RoundingMode::TowardNegative)
+ if (RoundMode != (int)RoundingMode::TowardZero &&
+ RoundMode != (int)RoundingMode::TowardNegative &&
+ RoundMode != (int)RoundingMode::TowardPositive &&
+ RoundMode != (int)RoundingMode::TowardNegative)
return false;
// "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5316bd077f55f7..a8e16d6b7e40fc 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6662,8 +6662,10 @@ SDValue SITargetLowering::lowerFPTRUNC_ROUND(SDValue Op,
// Only support towardzero, tonearest, upward and downward.
int RoundMode = Op.getConstantOperandVal(1);
- if (RoundMode < (int)RoundingMode::TowardZero ||
- RoundMode > (int)RoundingMode::TowardNegative)
+ if (RoundMode != (int)RoundingMode::TowardZero &&
+ RoundMode != (int)RoundingMode::TowardNegative &&
+ RoundMode != (int)RoundingMode::TowardPositive &&
+ RoundMode != (int)RoundingMode::TowardNegative)
return SDValue();
// "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
@@ -6681,8 +6683,6 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
assert(Op.getValueType() == MVT::f16 &&
"Do not know how to custom lower FP_ROUND for non-f16 type");
- return Op;
-
SDValue Src = Op.getOperand(0);
EVT SrcVT = Src.getValueType();
if (SrcVT != MVT::f64)
More information about the llvm-commits
mailing list