[llvm] [AMDGPU] Adjust amdgpu fmax/fmin legalization (PR #202917)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 03:24:09 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Shoreshen
<details>
<summary>Changes</summary>
---
Patch is 1.14 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/202917.diff
38 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPU.td (+4-5)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInstructions.td (+14-6)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+29-24)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp (+12-2)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp (+32-3)
- (modified) llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp (+19-4)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+79-30)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+117)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.h (+2)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll (+33-49)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll (+33-49)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll (-8)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll (-18)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll (+2-46)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll (+3-6)
- (modified) llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll (+84-108)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll (+126-162)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll (+126-162)
- (modified) llvm/test/CodeGen/AMDGPU/clamp.ll (+1-8)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll (+294-417)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll (+294-417)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll (+32-66)
- (modified) llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll (+14-46)
- (modified) llvm/test/CodeGen/AMDGPU/fmaxnum.ll (+176-458)
- (modified) llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll (+14-46)
- (modified) llvm/test/CodeGen/AMDGPU/fmin3.ll (+32-42)
- (modified) llvm/test/CodeGen/AMDGPU/fminnum.ll (+164-431)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll (+234-341)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll (+234-341)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll (+34-62)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll (+34-62)
- (modified) llvm/test/CodeGen/AMDGPU/maximumnum.ll (+755-2413)
- (modified) llvm/test/CodeGen/AMDGPU/minimumnum.ll (+726-2328)
- (modified) llvm/test/CodeGen/AMDGPU/minmax.ll (+52-194)
- (modified) llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll (+10-24)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-fp16.ll (+35-124)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-fmax.ll (+305-760)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-fmin.ll (+303-758)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index db375bc8803a1..26ce9e8d8e09a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -177,12 +177,13 @@ defm FmaMixBF16Insts : AMDGPUSubtargetFeature<"fma-mix-bf16-insts",
>;
defm IEEEMinimumMaximumInsts : AMDGPUSubtargetFeature<"ieee-minimum-maximum-insts",
- "Has v_minimum/maximum_f16/f32/f64, v_minimummaximum/maximumminimum_f16/f32 and"
- "v_pk_minimum/maximum_f16 instructions"
+ "Has v_minimum/maximum_f16/f32/f64, v_minimummaximum/maximumminimum_f16/f32,"
+ "v_pk_minmax/maxmin_num_f16, v_min/max_num_f16/f32/f64,"
+ "and v_pk_min/max_num_f16 instructions"
>;
defm SALUMinimumMaximumInsts : AMDGPUSubtargetFeature<"salu-minimum-maximum-insts",
- "Has s_minimum/maximum_f16/f32 instructions"
+ "Has s_minimum/maximum_f16/f32 and s_max/min_num_f16/f32 instructions"
>;
defm Minimum3Maximum3F32 : AMDGPUSubtargetFeature<"minimum3-maximum3-f32",
@@ -2075,7 +2076,6 @@ def FeatureISAVersion12 : FeatureSet<
FeatureFP8ConversionInsts,
FeatureWMMA128bInsts,
FeatureSWMMACGfx1200Insts,
- FeatureIEEEMinimumMaximumInsts,
FeaturePackedTID,
FeatureVcmpxPermlaneHazard,
FeatureSALUFloatInsts,
@@ -2287,7 +2287,6 @@ def FeatureISAVersion13 : FeatureSet<
FeatureGloballyAddressableScratch,
FeatureCvtPkF16F32Inst,
FeatureF16BF16ToFP6BF6ConversionScaleInsts,
- FeatureIEEEMinimumMaximumInsts,
FeatureSWakeupBarrier,
FeatureClusters,
FeatureCubeInsts,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 11d8aef61c858..f34dbfe4a5b46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -249,6 +249,8 @@ def umin_oneuse : HasOneUseBinOp<umin>;
def fminnum_oneuse : HasOneUseBinOp<fminnum>;
def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
+def fminimumnum_oneuse : HasOneUseBinOp<fminimumnum>;
+def fmaximumnum_oneuse : HasOneUseBinOp<fmaximumnum>;
def fminimum_oneuse : HasOneUseBinOp<fminimum>;
def fmaximum_oneuse : HasOneUseBinOp<fmaximum>;
@@ -847,13 +849,15 @@ class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat <
// Instructions which select to the same v_min_f*
def fminnum_like : PatFrags<(ops node:$src0, node:$src1),
[(fminnum_ieee node:$src0, node:$src1),
- (fminnum node:$src0, node:$src1)]
+ (fminnum node:$src0, node:$src1),
+ (fminimumnum node:$src0, node:$src1)]
>;
// Instructions which select to the same v_max_f*
def fmaxnum_like : PatFrags<(ops node:$src0, node:$src1),
[(fmaxnum_ieee node:$src0, node:$src1),
- (fmaxnum node:$src0, node:$src1)]
+ (fmaxnum node:$src0, node:$src1),
+ (fmaximumnum node:$src0, node:$src1)]
>;
class NeverNaNPats<dag ops, list<dag> frags> : PatFrags<ops, frags> {
@@ -867,22 +871,26 @@ class NeverNaNPats<dag ops, list<dag> frags> : PatFrags<ops, frags> {
def fminnum_like_nnan : NeverNaNPats<(ops node:$src0, node:$src1),
[(fminnum_ieee node:$src0, node:$src1),
- (fminnum node:$src0, node:$src1)]
+ (fminnum node:$src0, node:$src1),
+ (fminimumnum node:$src0, node:$src1)]
>;
def fmaxnum_like_nnan : NeverNaNPats<(ops node:$src0, node:$src1),
[(fmaxnum_ieee node:$src0, node:$src1),
- (fmaxnum node:$src0, node:$src1)]
+ (fmaxnum node:$src0, node:$src1),
+ (fmaximumnum node:$src0, node:$src1)]
>;
def fminnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
[(fminnum_ieee_oneuse node:$src0, node:$src1),
- (fminnum_oneuse node:$src0, node:$src1)]
+ (fminnum_oneuse node:$src0, node:$src1),
+ (fminimumnum_oneuse node:$src0, node:$src1)]
>;
def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
[(fmaxnum_ieee_oneuse node:$src0, node:$src1),
- (fmaxnum_oneuse node:$src0, node:$src1)]
+ (fmaxnum_oneuse node:$src0, node:$src1),
+ (fmaximumnum_oneuse node:$src0, node:$src1)]
>;
def any_fmad : PatFrags<(ops node:$src0, node:$src1, node:$src2),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e341392ae068a..40896d3af7580 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -983,38 +983,40 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &MinNumMaxNumIeee =
getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
+ auto &MinNumMaxNum = getActionDefinitionsBuilder(
+ {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
+ bool HasMinnumMaxnum =
+ ST.hasIEEEMinimumMaximumInsts() || ST.hasSALUMinimumMaximumInsts();
if (ST.hasVOP3PInsts()) {
- MinNumMaxNumIeee.legalFor(FPTypesPK16)
+ MinNumMaxNumIeee.legalFor(!HasMinnumMaxnum, FPTypesPK16)
+ .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
+ .clampMaxNumElements(0, S16, 2)
+ .clampScalar(0, S16, S64)
+ .scalarize(0);
+
+ MinNumMaxNum.legalFor(HasMinnumMaxnum, FPTypesPK16)
+ .customFor(!HasMinnumMaxnum, FPTypesPK16)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.clampMaxNumElements(0, S16, 2)
.clampScalar(0, S16, S64)
.scalarize(0);
} else if (ST.has16BitInsts()) {
- MinNumMaxNumIeee.legalFor(FPTypes16).clampScalar(0, S16, S64).scalarize(0);
+ MinNumMaxNumIeee.legalFor(!HasMinnumMaxnum, FPTypes16)
+ .clampScalar(0, S16, S64)
+ .scalarize(0);
+ MinNumMaxNum.legalFor(HasMinnumMaxnum, FPTypes16)
+ .customFor(!HasMinnumMaxnum, FPTypes16)
+ .clampScalar(0, S16, S64)
+ .scalarize(0);
} else {
- MinNumMaxNumIeee.legalFor(FPTypesBase)
+ MinNumMaxNumIeee.legalFor(!HasMinnumMaxnum, FPTypesBase)
+ .clampScalar(0, S32, S64)
+ .scalarize(0);
+ MinNumMaxNum.legalFor(HasMinnumMaxnum, FPTypesBase)
+ .customFor(!HasMinnumMaxnum, FPTypesBase)
.clampScalar(0, S32, S64)
.scalarize(0);
- }
-
- auto &MinNumMaxNum = getActionDefinitionsBuilder(
- {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
-
- if (ST.hasVOP3PInsts()) {
- MinNumMaxNum.customFor(FPTypesPK16)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .clampMaxNumElements(0, S16, 2)
- .clampScalar(0, S16, S64)
- .scalarize(0);
- } else if (ST.has16BitInsts()) {
- MinNumMaxNum.customFor(FPTypes16)
- .clampScalar(0, S16, S64)
- .scalarize(0);
- } else {
- MinNumMaxNum.customFor(FPTypesBase)
- .clampScalar(0, S32, S64)
- .scalarize(0);
}
if (ST.hasVOP3PInsts())
@@ -4336,7 +4338,8 @@ bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
// We don't need to concern ourselves with the snan handling difference, so
// use the one which will directly select.
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
- if (MFI->getMode().IEEE)
+ if (MFI->getMode().IEEE && !ST.hasIEEEMinimumMaximumInsts() &&
+ !ST.hasSALUMinimumMaximumInsts())
B.buildFMinNumIEEE(Min, Fract, Const, Flags);
else
B.buildFMinNum(Min, Fract, Const, Flags);
@@ -6133,7 +6136,9 @@ bool AMDGPULegalizerInfo::legalizeRsqClampIntrinsic(MachineInstr &MI,
// We don't need to concern ourselves with the snan handling difference, since
// the rsq quieted (or not) so use the one which will directly select.
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
- const bool UseIEEE = MFI->getMode().IEEE;
+ const bool UseIEEE = MFI->getMode().IEEE &&
+ !ST.hasIEEEMinimumMaximumInsts() &&
+ !ST.hasSALUMinimumMaximumInsts();
auto MaxFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics));
auto ClampMax = UseIEEE ? B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags) :
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
index 74184333e5b69..2eb9fecffcbbd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
@@ -108,6 +108,8 @@ class AMDGPURegBankCombinerImpl : public Combiner {
bool getIEEE() const;
bool getDX10Clamp() const;
bool isFminnumIeee(const MachineInstr &MI) const;
+ bool isFminnum(const MachineInstr &MI) const;
+ bool isFminnumLike(const MachineInstr &MI) const;
bool isFCst(MachineInstr *MI) const;
bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1) const;
@@ -275,7 +277,7 @@ bool AMDGPURegBankCombinerImpl::matchFPMinMaxToMed3(
// nodes(max/min) have same behavior when one input is NaN and other isn't.
// Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN,
// also post-legalizer inputs to min/max are fcanonicalized (never SNaN).
- if ((getIEEE() && isFminnumIeee(MI)) || VT->isKnownNeverNaN(Dst)) {
+ if ((getIEEE() && isFminnumLike(MI)) || VT->isKnownNeverNaN(Dst)) {
// Don't fold single use constant that can't be inlined.
if ((!MRI.hasOneNonDBGUse(K0->VReg) || TII.isInlineConstant(K0->Value)) &&
(!MRI.hasOneNonDBGUse(K1->VReg) || TII.isInlineConstant(K1->Value))) {
@@ -304,7 +306,7 @@ bool AMDGPURegBankCombinerImpl::matchFPMinMaxToClamp(MachineInstr &MI,
// no NaN inputs. Most often MI is marked with nnan fast math flag.
// For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates
// to 0.0 requires dx10_clamp = true.
- if ((getIEEE() && getDX10Clamp() && isFminnumIeee(MI) &&
+ if ((getIEEE() && getDX10Clamp() && isFminnumLike(MI) &&
VT->isKnownNeverSNaN(Val)) ||
VT->isKnownNeverNaN(MI.getOperand(0).getReg())) {
Reg = Val;
@@ -587,6 +589,14 @@ bool AMDGPURegBankCombinerImpl::isFminnumIeee(const MachineInstr &MI) const {
return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE;
}
+bool AMDGPURegBankCombinerImpl::isFminnum(const MachineInstr &MI) const {
+ return MI.getOpcode() == AMDGPU::G_FMINNUM;
+}
+
+bool AMDGPURegBankCombinerImpl::isFminnumLike(const MachineInstr &MI) const {
+ return isFminnumIeee(MI) || isFminnum(MI);
+}
+
bool AMDGPURegBankCombinerImpl::isFCst(MachineInstr *MI) const {
return MI->getOpcode() == AMDGPU::G_FCONSTANT;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index f23524ee1fc6b..7d62ce2d63585 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -16,6 +16,7 @@
#include "AMDGPURegBankLegalizeRules.h"
#include "AMDGPUInstrInfo.h"
#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/MachineUniformityAnalysis.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -1546,9 +1547,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
- addRulesForGOpcs({G_FMINNUM_IEEE, G_FMAXNUM_IEEE, G_FMINNUM, G_FMAXNUM,
- G_FMINIMUMNUM, G_FMAXIMUMNUM},
- Standard)
+ // FMINNUM_IEEE/FMAXNUM_IEEE select to the IEEE scalar min/max (e.g.
+ // s_min_f32/s_max_f32), which is available whenever the target has the SALU
+ // float instructions, so a uniform value may map to SGPR.
+ addRulesForGOpcs({G_FMINNUM_IEEE, G_FMAXNUM_IEEE}, Standard)
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
.Uni(S64, {{UniInVgprS64}, {Vgpr64, Vgpr64}})
@@ -1560,6 +1562,33 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasSALUFloat)
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasSALUFloat);
+ auto ScalarNumIsSafe = [=](const MachineInstr &MI) {
+ const SIMachineFunctionInfo *MFI =
+ MI.getMF()->getInfo<SIMachineFunctionInfo>();
+ return hasSALUFloat && (!MFI->getMode().IEEE || hasSALUMinimumMaximumInsts);
+ };
+ auto ScalarNumIsNotSafe = [=](const MachineInstr &MI) {
+ return !ScalarNumIsSafe(MI);
+ };
+
+ // The _num_ min/max can use the mode-dependent scalar min/max outside IEEE
+ // mode. In IEEE mode, they only have a scalar form (e.g. s_max_num_f32) when
+ // the target has the SALU min/max-num instructions. Otherwise a uniform value
+ // must use the self-canonicalizing vector v_*_num form.
+ addRulesForGOpcs({G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM},
+ NoFastRules)
+ .Any({{DivS16}, {{Vgpr16}, {Vgpr16, Vgpr16}}})
+ .Any({{DivS32}, {{Vgpr32}, {Vgpr32, Vgpr32}}})
+ .Any({{UniS64}, {{UniInVgprS64}, {Vgpr64, Vgpr64}}})
+ .Any({{DivS64}, {{Vgpr64}, {Vgpr64, Vgpr64}}})
+ .Any({{UniV2S16}, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}}})
+ .Any({{DivV2S16}, {{VgprV2S16}, {VgprV2S16, VgprV2S16}}})
+ .Any({{{UniS16}, ScalarNumIsSafe}, {{Sgpr16}, {Sgpr16, Sgpr16}}})
+ .Any({{{UniS16}, ScalarNumIsNotSafe}, {{UniInVgprS16}, {Vgpr16, Vgpr16}}})
+ .Any({{{UniS32}, ScalarNumIsSafe}, {{Sgpr32}, {Sgpr32, Sgpr32}}})
+ .Any(
+ {{{UniS32}, ScalarNumIsNotSafe}, {{UniInVgprS32}, {Vgpr32, Vgpr32}}});
+
addRulesForGOpcs({G_FPTRUNC})
.Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})
.Any({{UniS32, S64}, {{UniInVgprS32}, {Vgpr64}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index cc1f2d0664484..9967592950683 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4085,10 +4085,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case AMDGPU::G_FFLOOR:
case AMDGPU::G_FCEIL:
case AMDGPU::G_INTRINSIC_ROUNDEVEN:
- case AMDGPU::G_FMINNUM:
- case AMDGPU::G_FMAXNUM:
- case AMDGPU::G_FMINIMUMNUM:
- case AMDGPU::G_FMAXIMUMNUM:
case AMDGPU::G_INTRINSIC_TRUNC:
case AMDGPU::G_STRICT_FADD:
case AMDGPU::G_STRICT_FSUB:
@@ -4101,6 +4097,25 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
return getDefaultMappingSOP(MI);
return getDefaultMappingVOP(MI);
}
+ case AMDGPU::G_FMINNUM:
+ case AMDGPU::G_FMAXNUM:
+ case AMDGPU::G_FMINIMUMNUM:
+ case AMDGPU::G_FMAXIMUMNUM: {
+ LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+ unsigned Size = Ty.getSizeInBits();
+ // Outside IEEE mode the mode-dependent scalar min/max (e.g. s_max_f32) has
+ // the correct numeric behavior, so a uniform op can map to SGPR. In IEEE
+ // mode those scalar ops would require explicit input canonicalization, so
+ // only map to SGPR when a scalar _num_ min/max (e.g. s_max_num_f32) exists;
+ // otherwise force VGPR to use the self-canonicalizing v_*_num_f* form.
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ bool ScalarNumIsSafe =
+ !MFI->getMode().IEEE || Subtarget.hasSALUMinimumMaximumInsts();
+ if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
+ (Size == 32 || Size == 16) && isSALUMapping(MI) && ScalarNumIsSafe)
+ return getDefaultMappingSOP(MI);
+ return getDefaultMappingVOP(MI);
+ }
case AMDGPU::G_FMINIMUM:
case AMDGPU::G_FMAXIMUM: {
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a6dc58e6da263..ce776dba837f9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -541,15 +541,35 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
Legal);
- setOperationAction(
- {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
- {MVT::f32, MVT::f64}, Custom);
-
- // These are really only legal for ieee_mode functions. We should be avoiding
- // them for functions that don't have ieee_mode enabled, so just say they are
- // legal.
- setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
- {MVT::f32, MVT::f64}, Legal);
+ // Do not have s_{min|max}_*f64 instruction f64 will only be lowered to
+ // v_{min|max}_*f64
+ if (Subtarget->hasIEEEMinimumMaximumInsts()) {
+ setOperationAction(
+ {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f64, Legal);
+ } else {
+ setOperationAction(
+ {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f64, Custom);
+ // These are really only legal for ieee_mode functions. We should be
+ // avoiding them for functions that don't have ieee_mode enabled, so just
+ // say they are legal.
+ setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, MVT::f64, Legal);
+ }
+
+ // If has v_{min|max}_num_f32 but no s_{min|max}_num_f32, leave it for custom
+ // function
+ if (Subtarget->hasIEEEMinimumMaximumInsts() &&
+ Subtarget->hasSALUMinimumMaximumInsts()) {
+ setOperationAction(
+ {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f32, Legal);
+ } else {
+ setOperationAction(
+ {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f32, Custom);
+ setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, MVT::f32, Legal);
+ }
if (Subtarget->haveRoundOpsF64())
setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, MVT::f64,
@@ -818,20 +838,32 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
MVT::v32f16, MVT::v32bf16},
Custom);
+ if (Subtarget->hasIEEEMinimumMaximumInsts() &&
+ Subtarget->hasSALUMinimumMaximumInsts()) {
+ setOperationAction(
+ {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f16, Legal);
- setOperationAction(
- {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
- MVT::f16, Custom);
- setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);
+ setOperationAction(
+ {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, Custom);
+ } else {
+ setOperationAction(
+ {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::f16, Custom);
- setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM,
- ISD::FMAXIMUMNUM},
- {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
- Custom);
+ setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16,
+ Legal);
- setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
- {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
- Expand);
+ setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
+ ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
+ Custom);
+
+ setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
+ {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
+ Expand);
+ }
for (MVT Vec16 :
{MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
@@ -850,7 +882,6 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
MVT::v2i16, Legal);
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
- ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
ISD::FCANONICALIZE},
MVT::v2f16, Legal);
@@ -877,23 +908,33 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::FCANONICALIZE},
VT, Custom);
- setOperationAction(
- {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
- {MVT::v2f16, MVT::v4f16}, Custom);
+ if (Subtarget->hasIEEEMinimumMaximumInsts()) {
+ setOperationAction(
+ {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
+ MVT::v2f16, Legal);
+ } ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/202917
More information about the llvm-commits
mailing list