[llvm] Intrinsic: Fix minnum and introduce minimumnum (PR #93373)
YunQiang Su via llvm-commits
llvm-commits at lists.llvm.org
Sat May 25 00:42:25 PDT 2024
https://github.com/wzssyqa created https://github.com/llvm/llvm-project/pull/93373
Currently, on different platform, the behaivor of llvm.minnum is different if one operand is sNaN.
When we compute sNaN vs NUM
ARM/AArch64 follow the IEEE754-2008's minNUM: return qNaN.
LIBCALL: returns qNaN.
RV/X86/Hexagon follow the IEEE754-2019's minimumNumber: return NUM.
MIPS/LoongArch/Generic: return NUM.
So, let's introduce `llvm.minmumnum/llvm.maximumnum`, which always follow IEEE754-2019's minimumNumber/maximumNumber.
Since `llvm.minnum` shares the same name with IEEE754-2008's minNUM, and currently, it is used for fmin(3), we should ask all ports to switch the behavior to minNUM.
Fixes: #93033
>From 13ff1371e22f45d52529e62a977b44d2f605fab6 Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq at gcc.gnu.org>
Date: Sat, 25 May 2024 15:27:02 +0800
Subject: [PATCH] Intrinsic: Fix minnum and introduce minimumnum
Currently, on different platform, the behaivor of llvm.minnum is
different if one operand is sNaN.
When we compute sNaN vs NUM
ARM/AArch64 follow the IEEE754-2008's minNUM: return qNaN.
LIBCALL: returns qNaN.
RV/X86/Hexagon follow the IEEE754-2019's minimumNumber: return NUM.
MIPS/LoongArch/Generic: return NUM.
So, let's introduce `llvm.minmumnum/llvm.maximumnum`, which always
follow IEEE754-2019's minimumNumber/maximumNumber.
Since `llvm.minnum` shares the same name with IEEE754-2008's minNUM,
and currently, it is used for fmin(3), we should ask all ports to
switch the behavior to minNUM.
Fixes: #93033
---
llvm/include/llvm/ADT/APFloat.h | 28 +++-
llvm/include/llvm/Analysis/IVDescriptors.h | 5 +-
.../llvm/Analysis/TargetLibraryInfo.def | 68 ++++++++-
llvm/include/llvm/Analysis/ValueTracking.h | 4 +
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 10 ++
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 2 +
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 12 ++
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 +
.../CodeGen/GlobalISel/MachineIRBuilder.h | 36 +++++
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 4 +
llvm/include/llvm/CodeGen/ISDOpcodes.h | 12 ++
llvm/include/llvm/CodeGen/TargetLowering.h | 7 +
llvm/include/llvm/IR/ConstrainedOps.def | 2 +
llvm/include/llvm/IR/IRBuilder.h | 10 ++
llvm/include/llvm/IR/IntrinsicInst.h | 2 +
llvm/include/llvm/IR/Intrinsics.td | 40 +++++
llvm/include/llvm/IR/RuntimeLibcalls.def | 20 +++
llvm/include/llvm/IR/VPIntrinsics.def | 22 +++
llvm/include/llvm/Support/TargetOpcodes.def | 6 +
llvm/include/llvm/Target/GenericOpcodes.td | 19 +++
.../include/llvm/Target/GlobalISel/Combine.td | 6 +-
.../Target/GlobalISel/SelectionDAGCompat.td | 4 +
.../include/llvm/Target/TargetSelectionDAG.td | 18 +++
llvm/lib/Analysis/ConstantFolding.cpp | 8 +
llvm/lib/Analysis/IVDescriptors.cpp | 4 +
llvm/lib/Analysis/InstructionSimplify.cpp | 17 ++-
llvm/lib/Analysis/ValueTracking.cpp | 24 ++-
llvm/lib/Analysis/VectorUtils.cpp | 2 +
llvm/lib/CodeGen/ExpandVectorPredication.cpp | 20 ++-
llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 2 +
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 2 +
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 8 +
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 58 +++++++-
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 18 +++
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 +-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 49 +++++-
.../SelectionDAG/LegalizeFloatTypes.cpp | 104 ++++++++++++-
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 8 +
.../SelectionDAG/LegalizeVectorOps.cpp | 13 ++
.../SelectionDAG/LegalizeVectorTypes.cpp | 22 ++-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 20 ++-
.../SelectionDAG/SelectionDAGBuilder.cpp | 38 +++++
.../SelectionDAG/SelectionDAGDumper.cpp | 6 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 140 +++++++++++++++---
llvm/lib/CodeGen/TargetLoweringBase.cpp | 1 +
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 8 +
.../Target/Hexagon/HexagonISelLowering.cpp | 6 +-
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 6 +
llvm/lib/Target/Hexagon/HexagonPatterns.td | 4 +
llvm/lib/Target/Hexagon/HexagonPatternsHVX.td | 8 +
.../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 2 +-
.../RISCV/GISel/RISCVRegisterBankInfo.cpp | 4 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 27 +++-
llvm/lib/Target/RISCV/RISCVInstrInfoD.td | 2 +
llvm/lib/Target/RISCV/RISCVInstrInfoF.td | 2 +
.../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 2 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 10 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 11 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 4 +
60 files changed, 937 insertions(+), 74 deletions(-)
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index deb74cb2fdeb1..69dcb3d5f2634 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -1422,9 +1422,19 @@ inline APFloat maxnum(const APFloat &A, const APFloat &B) {
LLVM_READONLY
inline APFloat minimum(const APFloat &A, const APFloat &B) {
if (A.isNaN())
- return A;
+ return A.makeQuiet();
if (B.isNaN())
- return B;
+ return B.makeQuiet();
+ if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
+ return A.isNegative() ? A : B;
+ return B < A ? B : A;
+}
+LLVM_READONLY
+inline APFloat minimumnum(const APFloat &A, const APFloat &B) {
+ if (A.isNaN())
+ return B.isNaN() ? B.makeQuiet() : B;
+ if (B.isNaN())
+ return A;
if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
return A.isNegative() ? A : B;
return B < A ? B : A;
@@ -1435,9 +1445,19 @@ inline APFloat minimum(const APFloat &A, const APFloat &B) {
LLVM_READONLY
inline APFloat maximum(const APFloat &A, const APFloat &B) {
if (A.isNaN())
- return A;
+ return A.makeQuiet();
if (B.isNaN())
- return B;
+ return B.makeQuiet();
+ if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
+ return A.isNegative() ? B : A;
+ return A < B ? B : A;
+}
+LLVM_READONLY
+inline APFloat maximumnum(const APFloat &A, const APFloat &B) {
+ if (A.isNaN())
+ return B.isNaN() ? B.makeQuiet() : B;
+ if (B.isNaN())
+ return A;
if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
return A.isNegative() ? B : A;
return A < B ? B : A;
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 5c7b613ac48c4..cba3e568daca4 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -48,6 +48,8 @@ enum class RecurKind {
FMax, ///< FP max implemented in terms of select(cmp()).
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
+ FMinimumnum, ///< FP min with llvm.minimumnum semantics
+ FMaximumnum, ///< FP max with llvm.maximumnum semantics
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
///< loop invariant, and both x and y are integer type.
@@ -226,7 +228,8 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating-point min/max kind.
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
- Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum;
+ Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
+ Kind == RecurKind::FMinimumnum || Kind == RecurKind::FMaximumnum;
}
/// Returns true if the recurrence kind is any min/max kind.
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index 717693a7cf63c..d91891a852618 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1314,7 +1314,7 @@ TLI_DEFINE_ENUM_INTERNAL(flsll)
TLI_DEFINE_STRING_INTERNAL("flsll")
TLI_DEFINE_SIG_INTERNAL(Int, LLong)
-// Calls to fmax and fmin library functions expand to the llvm.maxnnum and
+// Calls to fmax and fmin library functions expand to the llvm.maxnum and
// llvm.minnum intrinsics with the correct parameter types for the arguments
// (all types must match).
/// double fmax(double x, double y);
@@ -1347,6 +1347,72 @@ TLI_DEFINE_ENUM_INTERNAL(fminl)
TLI_DEFINE_STRING_INTERNAL("fminl")
TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+// Calls to fmaximum and fminimum library functions expand to the llvm.maximum and
+// llvm.minimum intrinsics with the correct parameter types for the arguments
+// (all types must match).
+/// double fmaximum(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum)
+TLI_DEFINE_STRING_INTERNAL("fmaximum")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fmaximumf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximumf)
+TLI_DEFINE_STRING_INTERNAL("fmaximumf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fmaximuml(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximuml)
+TLI_DEFINE_STRING_INTERNAL("fmaximuml")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// double fminimum(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum)
+TLI_DEFINE_STRING_INTERNAL("fminimum")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fminimumf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fminimumf)
+TLI_DEFINE_STRING_INTERNAL("fminimumf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fminimuml(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimuml)
+TLI_DEFINE_STRING_INTERNAL("fminimuml")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+// Calls to fmaximum_num and fminimum_num library functions expand to the llvm.maximumnum and
+// llvm.minimumnum intrinsics with the correct parameter types for the arguments
+// (all types must match).
+/// double fmaximum_num(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_num)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_num")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fmaximum_numf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_numf)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_numf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fmaximum_numl(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_numl)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_numl")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// double fminimum_num(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_num)
+TLI_DEFINE_STRING_INTERNAL("fminimum_num")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fminimum_numf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_numf)
+TLI_DEFINE_STRING_INTERNAL("fminimum_numf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fminimum_numl(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_numl)
+TLI_DEFINE_STRING_INTERNAL("fminimum_numl")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
/// double fmod(double x, double y);
TLI_DEFINE_ENUM_INTERNAL(fmod)
TLI_DEFINE_STRING_INTERNAL("fmod")
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 0584b7e29f67b..c8aaa9d75c119 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -1065,6 +1065,10 @@ enum SelectPatternFlavor {
SPF_UMAX, /// Unsigned maximum
SPF_FMINNUM, /// Floating point minnum
SPF_FMAXNUM, /// Floating point maxnum
+ SPF_FMINIMUM,/// Floating point minnum
+ SPF_FMAXIMUM,/// Floating point maxnum
+ SPF_FMINIMUMNUM, /// Floating point minnum
+ SPF_FMAXIMUMNUM, /// Floating point maxnum
SPF_ABS, /// Absolute value
SPF_NABS /// Negated absolute value
};
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 2091432d4fe27..167cc2d1755a5 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1686,6 +1686,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::vector_reduce_fmin:
case Intrinsic::vector_reduce_fmaximum:
case Intrinsic::vector_reduce_fminimum:
+ case Intrinsic::vector_reduce_fmaximumnum:
+ case Intrinsic::vector_reduce_fminimumnum:
case Intrinsic::vector_reduce_umax:
case Intrinsic::vector_reduce_umin: {
IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, I, 1);
@@ -2009,6 +2011,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::maximum:
ISD = ISD::FMAXIMUM;
break;
+ case Intrinsic::minimumnum:
+ ISD = ISD::FMINIMUMNUM;
+ break;
+ case Intrinsic::maximumnum:
+ ISD = ISD::FMAXIMUMNUM;
+ break;
case Intrinsic::copysign:
ISD = ISD::FCOPYSIGN;
break;
@@ -2090,6 +2098,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::vector_reduce_fmin:
case Intrinsic::vector_reduce_fmaximum:
case Intrinsic::vector_reduce_fminimum:
+ case Intrinsic::vector_reduce_fmaximumnum:
+ case Intrinsic::vector_reduce_fminimumnum:
return thisT()->getMinMaxReductionCost(getMinMaxReductionIntrinsicOp(IID),
VecOpTy, ICA.getFlags(), CostKind);
case Intrinsic::abs: {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 2111e82e1a99d..c6fe31b3cd4fe 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -947,8 +947,10 @@ class CombinerHelper {
///
/// * G_FMAXNUM
/// * G_FMAXIMUM
+ /// * G_FMAXIMUMNUM
/// * G_FMINNUM
/// * G_FMINIMUM
+ /// * G_FMINIMUMNUM
///
/// Helper function for matchFPSelectToMinMax.
unsigned getFPMinMaxOpcForSelect(CmpInst::Predicate Pred, LLT DstTy,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 2b3efc3b609f0..47535559e77f1 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -551,6 +551,8 @@ class GVecReduce : public GenericMachineInstr {
case TargetOpcode::G_VECREDUCE_FMIN:
case TargetOpcode::G_VECREDUCE_FMAXIMUM:
case TargetOpcode::G_VECREDUCE_FMINIMUM:
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
case TargetOpcode::G_VECREDUCE_ADD:
case TargetOpcode::G_VECREDUCE_MUL:
case TargetOpcode::G_VECREDUCE_AND:
@@ -589,6 +591,12 @@ class GVecReduce : public GenericMachineInstr {
case TargetOpcode::G_VECREDUCE_FMINIMUM:
ScalarOpc = TargetOpcode::G_FMINIMUM;
break;
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
+ ScalarOpc = TargetOpcode::G_FMAXIMUMNUM;
+ break;
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
+ ScalarOpc = TargetOpcode::G_FMINIMUMNUM;
+ break;
case TargetOpcode::G_VECREDUCE_ADD:
ScalarOpc = TargetOpcode::G_ADD;
break;
@@ -669,6 +677,8 @@ class GBinOp : public GenericMachineInstr {
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FADD:
case TargetOpcode::G_FSUB:
case TargetOpcode::G_FMUL:
@@ -719,6 +729,8 @@ class GFBinOp : public GBinOp {
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FADD:
case TargetOpcode::G_FSUB:
case TargetOpcode::G_FMUL:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 284f434fbb9b0..8ce6b44c7e1c6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -403,6 +403,7 @@ class LegalizerHelper {
LegalizeResult lowerMinMax(MachineInstr &MI);
LegalizeResult lowerFCopySign(MachineInstr &MI);
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI);
+ LegalizeResult lowerFMinimumNumMaximumNum(MachineInstr &MI);
LegalizeResult lowerFMad(MachineInstr &MI);
LegalizeResult lowerIntrinsicRound(MachineInstr &MI);
LegalizeResult lowerFFloor(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 92e05ee858a75..96cc0f738d258 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1723,6 +1723,30 @@ class MachineIRBuilder {
return buildInstr(TargetOpcode::G_FMAXNUM_IEEE, {Dst}, {Src0, Src1}, Flags);
}
+ MachineInstrBuilder
+ buildFMinimum(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+ std::optional<unsigned> Flags = std::nullopt) {
+ return buildInstr(TargetOpcode::G_FMINIMUM, {Dst}, {Src0, Src1}, Flags);
+ }
+
+ MachineInstrBuilder
+ buildFMaximum(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+ std::optional<unsigned> Flags = std::nullopt) {
+ return buildInstr(TargetOpcode::G_FMAXIMUM, {Dst}, {Src0, Src1}, Flags);
+ }
+
+ MachineInstrBuilder
+ buildFMinimumNUM(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+ std::optional<unsigned> Flags = std::nullopt) {
+ return buildInstr(TargetOpcode::G_FMINIMUMNUM, {Dst}, {Src0, Src1}, Flags);
+ }
+
+ MachineInstrBuilder
+ buildFMaximumNUM(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+ std::optional<unsigned> Flags = std::nullopt) {
+ return buildInstr(TargetOpcode::G_FMAXIMUMNUM, {Dst}, {Src0, Src1}, Flags);
+ }
+
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0,
const SrcOp &Src1,
std::optional<unsigned> Flags = std::nullopt) {
@@ -2074,6 +2098,18 @@ class MachineIRBuilder {
return buildInstr(TargetOpcode::G_VECREDUCE_FMINIMUM, {Dst}, {Src});
}
+ /// Build and insert \p Res = G_VECREDUCE_FMAXIMUMNUM \p Src
+ MachineInstrBuilder buildVecReduceFMaximumnum(const DstOp &Dst,
+ const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_VECREDUCE_FMAXIMUMNUM, {Dst}, {Src});
+ }
+
+ /// Build and insert \p Res = G_VECREDUCE_FMINIMUMNUM \p Src
+ MachineInstrBuilder buildVecReduceFMinimumnum(const DstOp &Dst,
+ const SrcOp &Src) {
+ return buildInstr(TargetOpcode::G_VECREDUCE_FMINIMUMNUM, {Dst}, {Src});
+ }
+
/// Build and insert \p Res = G_VECREDUCE_ADD \p Src
MachineInstrBuilder buildVecReduceAdd(const DstOp &Dst, const SrcOp &Src) {
return buildInstr(TargetOpcode::G_VECREDUCE_ADD, {Dst}, {Src});
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 70421a518ab72..c26f8ffd839e7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -60,6 +60,8 @@ class APFloat;
case TargetOpcode::G_VECREDUCE_FMIN: \
case TargetOpcode::G_VECREDUCE_FMAXIMUM: \
case TargetOpcode::G_VECREDUCE_FMINIMUM: \
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM: \
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM: \
case TargetOpcode::G_VECREDUCE_ADD: \
case TargetOpcode::G_VECREDUCE_MUL: \
case TargetOpcode::G_VECREDUCE_AND: \
@@ -77,6 +79,8 @@ class APFloat;
case TargetOpcode::G_VECREDUCE_FMIN: \
case TargetOpcode::G_VECREDUCE_FMAXIMUM: \
case TargetOpcode::G_VECREDUCE_FMINIMUM: \
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM: \
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM: \
case TargetOpcode::G_VECREDUCE_ADD: \
case TargetOpcode::G_VECREDUCE_MUL: \
case TargetOpcode::G_VECREDUCE_AND: \
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index d8af97957e48e..910728e38c1b1 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -435,6 +435,8 @@ enum NodeType {
STRICT_LLRINT,
STRICT_FMAXIMUM,
STRICT_FMINIMUM,
+ STRICT_FMAXIMUMNUM,
+ STRICT_FMINIMUMNUM,
/// STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or
/// unsigned integer. These have the same semantics as fptosi and fptoui
@@ -991,6 +993,12 @@ enum NodeType {
FMINIMUM,
FMAXIMUM,
+ /// FMINIMUMNUM/FMAXIMUMNUM - minimum/maximum that also treat -0.0
+ /// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008
+ /// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2019 semantics.
+ FMINIMUMNUM,
+ FMAXIMUMNUM,
+
/// FSINCOS - Compute both fsin and fcos as a single operation.
FSINCOS,
@@ -1365,6 +1373,10 @@ enum NodeType {
/// llvm.minimum and llvm.maximum semantics.
VECREDUCE_FMAXIMUM,
VECREDUCE_FMINIMUM,
+ /// FMINIMUMNUM/FMAXIMUMNUM nodes doesn't propatate NaNs and signed zeroes using the
+ /// llvm.minimumnum and llvm.maximumnum semantics.
+ VECREDUCE_FMAXIMUMNUM,
+ VECREDUCE_FMINIMUMNUM,
/// Integer reductions may have a result type larger than the vector element
/// type. However, the reduction is performed using the vector element type
/// and the value in the top bits is unspecified.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 50a8c7eb75af5..c92eb86c08a48 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2908,6 +2908,8 @@ class TargetLoweringBase {
case ISD::FMAXNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::AVGFLOORS:
case ISD::AVGFLOORU:
case ISD::AVGCEILS:
@@ -5116,6 +5118,8 @@ class TargetLowering : public TargetLoweringBase {
/// through to the default expansion/soften to libcall, we might introduce a
/// link-time dependency on libm into a file that originally did not have one.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const;
+ SDValue createSelectForFMINIMUM_FMAXIMUM(SDNode *Node, SelectionDAG &DAG) const;
+ SDValue createSelectForFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node, SelectionDAG &DAG) const;
/// Return a reciprocal estimate value for the input operand.
/// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
@@ -5247,6 +5251,9 @@ class TargetLowering : public TargetLoweringBase {
/// Expand fminimum/fmaximum into multiple comparison with selects.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand fminimumnum/fmaximumnum into multiple comparison with selects.
+ SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
/// \param N Node to expand
/// \returns The expansion result
diff --git a/llvm/include/llvm/IR/ConstrainedOps.def b/llvm/include/llvm/IR/ConstrainedOps.def
index 41aa44de957f9..361a050cd2355 100644
--- a/llvm/include/llvm/IR/ConstrainedOps.def
+++ b/llvm/include/llvm/IR/ConstrainedOps.def
@@ -86,6 +86,8 @@ DAG_FUNCTION(maxnum, 2, 0, experimental_constrained_maxnum, FMAXNUM
DAG_FUNCTION(minnum, 2, 0, experimental_constrained_minnum, FMINNUM)
DAG_FUNCTION(maximum, 2, 0, experimental_constrained_maximum, FMAXIMUM)
DAG_FUNCTION(minimum, 2, 0, experimental_constrained_minimum, FMINIMUM)
+DAG_FUNCTION(maximumnum, 2, 0, experimental_constrained_maximumnum, FMAXIMUMNUM)
+DAG_FUNCTION(minimumnum, 2, 0, experimental_constrained_minimumnum, FMINIMUMNUM)
DAG_FUNCTION(nearbyint, 1, 1, experimental_constrained_nearbyint, FNEARBYINT)
DAG_FUNCTION(pow, 2, 1, experimental_constrained_pow, FPOW)
DAG_FUNCTION(powi, 2, 1, experimental_constrained_powi, FPOWI)
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 40a9cf507248a..34394c6478d16 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1018,6 +1018,16 @@ class IRBuilderBase {
return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
}
+ /// Create call to the minimumnum intrinsic.
+ Value *CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ return CreateBinaryIntrinsic(Intrinsic::minimumnum, LHS, RHS, nullptr, Name);
+ }
+
+ /// Create call to the maximum intrinsic.
+ Value *CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ return CreateBinaryIntrinsic(Intrinsic::maximumnum, LHS, RHS, nullptr, Name);
+ }
+
/// Create call to the copysign intrinsic.
Value *CreateCopySign(Value *LHS, Value *RHS,
Instruction *FMFSource = nullptr,
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index fcd3a1025ac13..caeaa3a13ea8a 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -75,6 +75,8 @@ class IntrinsicInst : public CallInst {
case Intrinsic::minnum:
case Intrinsic::maximum:
case Intrinsic::minimum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum:
case Intrinsic::smax:
case Intrinsic::smin:
case Intrinsic::umax:
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 3019f68083d42..01d6e27854e99 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1084,6 +1084,14 @@ def int_maximum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>],
[IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
>;
+def int_minimumnum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
+>;
+def int_maximumnum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
+>;
// Internal interface for object size checking
def int_objectsize : DefaultAttrsIntrinsic<[llvm_anyint_ty],
@@ -1274,6 +1282,14 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn, IntrStrictFP] in
[ LLVMMatchType<0>,
LLVMMatchType<0>,
llvm_metadata_ty ]>;
+ def int_experimental_constrained_maximumnum : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
+ [ LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ llvm_metadata_ty ]>;
+ def int_experimental_constrained_minimumnum : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
+ [ LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ llvm_metadata_ty ]>;
def int_experimental_constrained_ceil : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
[ LLVMMatchType<0>,
llvm_metadata_ty ]>;
@@ -2071,6 +2087,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
LLVMMatchType<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty]>;
+ def int_vp_minimumnum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+ [ LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty]>;
+ def int_vp_maximumnum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+ [ LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty]>;
def int_vp_copysign : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
[ LLVMMatchType<0>,
LLVMMatchType<0>,
@@ -2260,6 +2286,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
llvm_anyvector_ty,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty]>;
+ def int_vp_reduce_fmaximumnum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+ [ LLVMVectorElementType<0>,
+ llvm_anyvector_ty,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty]>;
+ def int_vp_reduce_fminimumnum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+ [ LLVMVectorElementType<0>,
+ llvm_anyvector_ty,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_i32_ty]>;
}
let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn, ImmArg<ArgIndex<1>>] in {
@@ -2498,6 +2534,10 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
[llvm_anyvector_ty]>;
def int_vector_reduce_fmaximum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
[llvm_anyvector_ty]>;
+ def int_vector_reduce_fminimumnum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+ [llvm_anyvector_ty]>;
+ def int_vector_reduce_fmaximumnum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+ [llvm_anyvector_ty]>;
}
//===----- Matrix intrinsics ---------------------------------------------===//
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 5e082769fa974..219214b6b0f00 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -264,6 +264,26 @@ HANDLE_LIBCALL(FMAX_F64, "fmax")
HANDLE_LIBCALL(FMAX_F80, "fmaxl")
HANDLE_LIBCALL(FMAX_F128, "fmaxl")
HANDLE_LIBCALL(FMAX_PPCF128, "fmaxl")
+HANDLE_LIBCALL(FMINIMUM_F32, "fminimumf")
+HANDLE_LIBCALL(FMINIMUM_F64, "fminimum")
+HANDLE_LIBCALL(FMINIMUM_F80, "fminimuml")
+HANDLE_LIBCALL(FMINIMUM_F128, "fminmuml")
+HANDLE_LIBCALL(FMINIMUM_PPCF128, "fminimuml")
+HANDLE_LIBCALL(FMAXIMUM_F32, "fmaximumf")
+HANDLE_LIBCALL(FMAXIMUM_F64, "fmaximum")
+HANDLE_LIBCALL(FMAXIMUM_F80, "fmaximuml")
+HANDLE_LIBCALL(FMAXIMUM_F128, "fmaxmuml")
+HANDLE_LIBCALL(FMAXIMUM_PPCF128, "fmaximuml")
+HANDLE_LIBCALL(FMINIMUMNUM_F32, "fminimum_numf")
+HANDLE_LIBCALL(FMINIMUMNUM_F64, "fminimum_num")
+HANDLE_LIBCALL(FMINIMUMNUM_F80, "fminimum_numl")
+HANDLE_LIBCALL(FMINIMUMNUM_F128, "fminmum_numl")
+HANDLE_LIBCALL(FMINIMUMNUM_PPCF128, "fminimum_numl")
+HANDLE_LIBCALL(FMAXIMUMNUM_F32, "fmaximum_numf")
+HANDLE_LIBCALL(FMAXIMUMNUM_F64, "fmaximum_num")
+HANDLE_LIBCALL(FMAXIMUMNUM_F80, "fmaximum_numl")
+HANDLE_LIBCALL(FMAXIMUMNUM_F128, "fmaxmum_numl")
+HANDLE_LIBCALL(FMAXIMUMNUM_PPCF128, "fmaximum_numl")
HANDLE_LIBCALL(LROUND_F32, "lroundf")
HANDLE_LIBCALL(LROUND_F64, "lround")
HANDLE_LIBCALL(LROUND_F80, "lroundl")
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index 20f5bb2b531d3..85b74bd0c0317 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -428,6 +428,20 @@ VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
+// llvm.vp.minimumnum(x,y,mask,vlen)
+BEGIN_REGISTER_VP(vp_minimumnum, 2, 3, VP_FMINIMUMNUM, -1)
+VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUMNUM)
+VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimumnum)
+END_REGISTER_VP(vp_minimumnum, VP_FMINIMUMNUM)
+
+// llvm.vp.maximumnum(x,y,mask,vlen)
+BEGIN_REGISTER_VP(vp_maximumnum, 2, 3, VP_FMAXIMUMNUM, -1)
+VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUMNUM)
+VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximumnum)
+END_REGISTER_VP(vp_maximumnum, VP_FMAXIMUMNUM)
+
// llvm.vp.ceil(x,mask,vlen)
BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)
@@ -709,6 +723,14 @@ HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
vector_reduce_fminimum)
+// llvm.vp.reduce.fmaximumnum(start,x,mask,vlen)
+HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximumnum, VP_REDUCE_FMAXIMUMNUM,
+ vector_reduce_fmaximumnum)
+
+// llvm.vp.reduce.fminimumnum(start,x,mask,vlen)
+HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimumnum, VP_REDUCE_FMINIMUMNUM,
+ vector_reduce_fminimumnum)
+
#undef HELPER_REGISTER_REDUCTION_VP
// Specialized helper macro for VP reductions as above but with two forms:
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 559a588c25148..92425bb4f00fa 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -692,6 +692,10 @@ HANDLE_TARGET_OPCODE(G_FMAXNUM_IEEE)
HANDLE_TARGET_OPCODE(G_FMINIMUM)
HANDLE_TARGET_OPCODE(G_FMAXIMUM)
+/// FP min_num/max_num matching IEEE-754 2019 semantics.
+HANDLE_TARGET_OPCODE(G_FMINIMUMNUM)
+HANDLE_TARGET_OPCODE(G_FMAXIMUMNUM)
+
/// Access to FP environment.
HANDLE_TARGET_OPCODE(G_GET_FPENV)
HANDLE_TARGET_OPCODE(G_SET_FPENV)
@@ -857,6 +861,8 @@ HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAX)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMIN)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAXIMUM)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMINIMUM)
+HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAXIMUMNUM)
+HANDLE_TARGET_OPCODE(G_VECREDUCE_FMINIMUMNUM)
HANDLE_TARGET_OPCODE(G_VECREDUCE_ADD)
HANDLE_TARGET_OPCODE(G_VECREDUCE_MUL)
HANDLE_TARGET_OPCODE(G_VECREDUCE_AND)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index c40498e554215..eca0a567eebaa 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -837,6 +837,23 @@ def G_FMAXIMUM : GenericInstruction {
let isCommutable = true;
}
+// FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that also treat -0.0
+// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008
+// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2019 semantics: if one operand
+// is SNaN, 2008 returns QNaN, while 2019 returns another non-NaN operand.
+def G_FMINIMUMNUM : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2);
+ let hasSideEffects = false;
+ let isCommutable = true;
+}
+
+def G_FMAXIMUMNUM : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type0:$src1, type0:$src2);
+ let hasSideEffects = false;
+ let isCommutable = true;
+}
//------------------------------------------------------------------------------
// Floating Point Binary ops.
//------------------------------------------------------------------------------
@@ -1529,6 +1546,8 @@ def G_VECREDUCE_FMAX : VectorReduction;
def G_VECREDUCE_FMIN : VectorReduction;
def G_VECREDUCE_FMAXIMUM : VectorReduction;
def G_VECREDUCE_FMINIMUM : VectorReduction;
+def G_VECREDUCE_FMAXIMUMNUM : VectorReduction;
+def G_VECREDUCE_FMINIMUMNUM : VectorReduction;
def G_VECREDUCE_ADD : VectorReduction;
def G_VECREDUCE_MUL : VectorReduction;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 34698f195615b..2c11155be916a 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -485,7 +485,7 @@ def commute_fp_constant_to_rhs : GICombineRule<
(defs root:$root),
(match (wip_match_opcode G_FADD, G_FMUL, G_FMINNUM, G_FMAXNUM,
G_FMINNUM_IEEE, G_FMAXNUM_IEEE,
- G_FMINIMUM, G_FMAXIMUM):$root,
+ G_FMINIMUM, G_FMAXIMUM, G_FMINIMUMNUM, G_FMAXIMUMNUM):$root,
[{ return Helper.matchCommuteFPConstantToRHS(*${root}); }]),
(apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
>;
@@ -582,7 +582,7 @@ def fold_binop_into_select : GICombineRule<
G_SDIV, G_SREM, G_UDIV, G_UREM, G_LSHR, G_ASHR, G_SHL,
G_SMIN, G_SMAX, G_UMIN, G_UMAX,
G_FMUL, G_FADD, G_FSUB, G_FDIV, G_FREM,
- G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root,
+ G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM, G_FMINIMUMNUM, G_FMAXIMUMNUM):$root,
[{ return Helper.matchFoldBinOpIntoSelect(*${root}, ${select_op_no}); }]),
(apply [{ Helper.applyFoldBinOpIntoSelect(*${root}, ${select_op_no}); }])
>;
@@ -1248,7 +1248,7 @@ def combine_fsub_fpext_fneg_fmul_to_fmad_or_fma: GICombineRule<
def combine_minmax_nan: GICombineRule<
(defs root:$root, unsigned_matchinfo:$info),
- (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root,
+ (match (wip_match_opcode G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM, G_FMINIMUMNUM, G_FMAXIMUMNUM):$root,
[{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]),
(apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${info}); }])>;
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 8fa0e4b86d6dc..e303acfbc701a 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -170,6 +170,8 @@ def : GINodeEquiv<G_FMINNUM_IEEE, fminnum_ieee>;
def : GINodeEquiv<G_FMAXNUM_IEEE, fmaxnum_ieee>;
def : GINodeEquiv<G_FMAXIMUM, fmaximum>;
def : GINodeEquiv<G_FMINIMUM, fminimum>;
+def : GINodeEquiv<G_FMAXIMUMNUM, fmaximumnum>;
+def : GINodeEquiv<G_FMINIMUMNUM, fminimumnum>;
def : GINodeEquiv<G_READCYCLECOUNTER, readcyclecounter>;
def : GINodeEquiv<G_READSTEADYCOUNTER, readsteadycounter>;
def : GINodeEquiv<G_ROTR, rotr>;
@@ -181,6 +183,8 @@ def : GINodeEquiv<G_VECREDUCE_FMAX, vecreduce_fmax>;
def : GINodeEquiv<G_VECREDUCE_FMIN, vecreduce_fmin>;
def : GINodeEquiv<G_VECREDUCE_FMAXIMUM, vecreduce_fmaximum>;
def : GINodeEquiv<G_VECREDUCE_FMINIMUM, vecreduce_fminimum>;
+def : GINodeEquiv<G_VECREDUCE_FMAXIMUMNUM, vecreduce_fmaximumnum>;
+def : GINodeEquiv<G_VECREDUCE_FMINIMUMNUM, vecreduce_fminimumnum>;
def : GINodeEquiv<G_VECREDUCE_UMIN, vecreduce_umin>;
def : GINodeEquiv<G_VECREDUCE_UMAX, vecreduce_umax>;
def : GINodeEquiv<G_VECREDUCE_SMIN, vecreduce_smin>;
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 1c95a60909846..7050562a1eda8 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -482,6 +482,8 @@ def vecreduce_fmin : SDNode<"ISD::VECREDUCE_FMIN", SDTFPVecReduce>;
def vecreduce_fmax : SDNode<"ISD::VECREDUCE_FMAX", SDTFPVecReduce>;
def vecreduce_fminimum : SDNode<"ISD::VECREDUCE_FMINIMUM", SDTFPVecReduce>;
def vecreduce_fmaximum : SDNode<"ISD::VECREDUCE_FMAXIMUM", SDTFPVecReduce>;
+def vecreduce_fminimumnum : SDNode<"ISD::VECREDUCE_FMINIMUMNUM", SDTFPVecReduce>;
+def vecreduce_fmaximumnum : SDNode<"ISD::VECREDUCE_FMAXIMUMNUM", SDTFPVecReduce>;
def fadd : SDNode<"ISD::FADD" , SDTFPBinOp, [SDNPCommutative]>;
def fsub : SDNode<"ISD::FSUB" , SDTFPBinOp>;
@@ -503,6 +505,10 @@ def fminimum : SDNode<"ISD::FMINIMUM" , SDTFPBinOp,
[SDNPCommutative, SDNPAssociative]>;
def fmaximum : SDNode<"ISD::FMAXIMUM" , SDTFPBinOp,
[SDNPCommutative, SDNPAssociative]>;
+def fminimumnum : SDNode<"ISD::FMINIMUMNUM" , SDTFPBinOp,
+ [SDNPCommutative, SDNPAssociative]>;
+def fmaximumnum : SDNode<"ISD::FMAXIMUMNUM" , SDTFPBinOp,
+ [SDNPCommutative, SDNPAssociative]>;
def fgetsign : SDNode<"ISD::FGETSIGN" , SDTFPToIntOp>;
def fcanonicalize : SDNode<"ISD::FCANONICALIZE", SDTFPUnaryOp>;
def fneg : SDNode<"ISD::FNEG" , SDTFPUnaryOp>;
@@ -604,6 +610,12 @@ def strict_fminimum : SDNode<"ISD::STRICT_FMINIMUM",
def strict_fmaximum : SDNode<"ISD::STRICT_FMAXIMUM",
SDTFPBinOp, [SDNPHasChain,
SDNPCommutative, SDNPAssociative]>;
+def strict_fminimumnum : SDNode<"ISD::STRICT_FMINIMUMNUM",
+ SDTFPBinOp, [SDNPHasChain,
+ SDNPCommutative, SDNPAssociative]>;
+def strict_fmaximumnum : SDNode<"ISD::STRICT_FMAXIMUMNUM",
+ SDTFPBinOp, [SDNPHasChain,
+ SDNPCommutative, SDNPAssociative]>;
def strict_fpround : SDNode<"ISD::STRICT_FP_ROUND",
SDTFPRoundOp, [SDNPHasChain]>;
def strict_fpextend : SDNode<"ISD::STRICT_FP_EXTEND",
@@ -1574,6 +1586,12 @@ def any_fmaximum : PatFrags<(ops node:$lhs, node:$rhs),
def any_fminimum : PatFrags<(ops node:$lhs, node:$rhs),
[(strict_fminimum node:$lhs, node:$rhs),
(fminimum node:$lhs, node:$rhs)]>;
+def any_fmaximumnum : PatFrags<(ops node:$lhs, node:$rhs),
+ [(strict_fmaximumnum node:$lhs, node:$rhs),
+ (fmaximumnum node:$lhs, node:$rhs)]>;
+def any_fminimumnum : PatFrags<(ops node:$lhs, node:$rhs),
+ [(strict_fminimumnum node:$lhs, node:$rhs),
+ (fminimumnum node:$lhs, node:$rhs)]>;
def any_fpround : PatFrags<(ops node:$src),
[(strict_fpround node:$src),
(fpround node:$src)]>;
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 31667ff3951f1..d2c84fb7e8ce9 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1549,6 +1549,8 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
case Intrinsic::maxnum:
case Intrinsic::minimum:
case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
case Intrinsic::log:
case Intrinsic::log2:
case Intrinsic::log10:
@@ -2605,6 +2607,8 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
case Intrinsic::minnum:
case Intrinsic::maximum:
case Intrinsic::minimum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum:
// If one argument is undef, return the other argument.
if (IsOp0Undef)
return Operands[1];
@@ -2668,6 +2672,10 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty->getContext(), minimum(Op1V, Op2V));
case Intrinsic::maximum:
return ConstantFP::get(Ty->getContext(), maximum(Op1V, Op2V));
+ case Intrinsic::minimumnum:
+ return ConstantFP::get(Ty->getContext(), minimumnum(Op1V, Op2V));
+ case Intrinsic::maximumnum:
+ return ConstantFP::get(Ty->getContext(), maximumnum(Op1V, Op2V));
}
if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 055f121e74341..7a82c261237ad 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -707,6 +707,10 @@ RecurrenceDescriptor::isMinMaxPattern(Instruction *I, RecurKind Kind,
return InstDesc(Kind == RecurKind::FMinimum, I);
if (match(I, m_Intrinsic<Intrinsic::maximum>(m_Value(), m_Value())))
return InstDesc(Kind == RecurKind::FMaximum, I);
+ if (match(I, m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())))
+ return InstDesc(Kind == RecurKind::FMinimumnum, I);
+ if (match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value())))
+ return InstDesc(Kind == RecurKind::FMaximumnum, I);
return InstDesc(false, I);
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 53a974c5294c6..8644e8fc5b4a0 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6339,7 +6339,8 @@ static Value *foldMinMaxSharedOp(Intrinsic::ID IID, Value *Op0, Value *Op1) {
static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
Value *Op1) {
assert((IID == Intrinsic::maxnum || IID == Intrinsic::minnum ||
- IID == Intrinsic::maximum || IID == Intrinsic::minimum) &&
+ IID == Intrinsic::maximum || IID == Intrinsic::minimum ||
+ IID == Intrinsic::maximumnum || IID == Intrinsic::minimumnum) &&
"Unsupported intrinsic");
auto *M0 = dyn_cast<IntrinsicInst>(Op0);
@@ -6352,6 +6353,8 @@ static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
Value *Y0 = M0->getOperand(1);
// Simple case, m(m(X,Y), X) => m(X, Y)
// m(m(X,Y), Y) => m(X, Y)
+ // For minimumnum/maximumnum, X is NaN => m(NaN, Y) == Y and m(Y, Y) == Y.
+ // For minimumnum/maximumnum, Y is NaN => m(X, NaN) == X and m(X, NaN) == X.
// For minimum/maximum, X is NaN => m(NaN, Y) == NaN and m(NaN, NaN) == NaN.
// For minimum/maximum, Y is NaN => m(X, NaN) == NaN and m(NaN, NaN) == NaN.
// For minnum/maxnum, X is NaN => m(NaN, Y) == Y and m(Y, Y) == Y.
@@ -6367,6 +6370,8 @@ static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
Intrinsic::ID IID1 = M1->getIntrinsicID();
// we have a case m(m(X,Y),m'(X,Y)) taking into account m' is commutative.
// if m' is m or inversion of m => m(m(X,Y),m'(X,Y)) == m(X,Y).
+ // For minimumnum/maximumnum, X is NaN => m(NaN,Y) == m'(NaN, Y) == Y.
+ // For minimumnum/maximumnum, Y is NaN => m(X,NaN) == m'(X, NaN) == X.
// For minimum/maximum, X is NaN => m(NaN,Y) == m'(NaN, Y) == NaN.
// For minimum/maximum, Y is NaN => m(X,NaN) == m'(X, NaN) == NaN.
// For minnum/maxnum, X is NaN => m(NaN,Y) == m'(NaN, Y) == Y.
@@ -6611,6 +6616,8 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
}
case Intrinsic::maxnum:
case Intrinsic::minnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum:
case Intrinsic::maximum:
case Intrinsic::minimum: {
// If the arguments are the same, this is a no-op.
@@ -6626,12 +6633,14 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
return Op0;
bool PropagateNaN = IID == Intrinsic::minimum || IID == Intrinsic::maximum;
- bool IsMin = IID == Intrinsic::minimum || IID == Intrinsic::minnum;
+ bool IsMin = IID == Intrinsic::minimum || IID == Intrinsic::minimumnum || IID == Intrinsic::minnum;
// minnum(X, nan) -> X
// maxnum(X, nan) -> X
// minimum(X, nan) -> nan
// maximum(X, nan) -> nan
+ // minimumnum(X, nan) -> X
+ // maximumnum(X, nan) -> X
if (match(Op1, m_NaN()))
return PropagateNaN ? propagateNaN(cast<Constant>(Op1)) : Op0;
@@ -6644,6 +6653,8 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
// maxnum(X, +inf) -> +inf
// minimum(X, -inf) -> -inf if nnan
// maximum(X, +inf) -> +inf if nnan
+ // minimumnum(X, -inf) -> -inf
+ // maximumnum(X, +inf) -> +inf
if (C->isNegative() == IsMin &&
(!PropagateNaN || (Call && Call->hasNoNaNs())))
return ConstantFP::get(ReturnType, *C);
@@ -6652,6 +6663,8 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
// maxnum(X, -inf) -> X if nnan
// minimum(X, +inf) -> X
// maximum(X, -inf) -> X
+ // minimumnum(X, +inf) -> X if nnan
+ // maximumnum(X, -inf) -> X if nnan
if (C->isNegative() != IsMin &&
(PropagateNaN || (Call && Call->hasNoNaNs())))
return Op0;
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 3baa8ede28ffa..2de6a48179e43 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4988,7 +4988,9 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
case Intrinsic::maxnum:
case Intrinsic::minnum:
case Intrinsic::minimum:
- case Intrinsic::maximum: {
+ case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum: {
KnownFPClass KnownLHS, KnownRHS;
computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedClasses,
KnownLHS, Depth + 1, Q);
@@ -5001,8 +5003,10 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
// If either operand is not NaN, the result is not NaN.
if (NeverNaN && (IID == Intrinsic::minnum || IID == Intrinsic::maxnum))
Known.knownNot(fcNan);
+ if (NeverNaN && (IID == Intrinsic::minimumnum || IID == Intrinsic::maximumnum))
+ Known.knownNot(fcNan);
- if (IID == Intrinsic::maxnum) {
+ if (IID == Intrinsic::maxnum || IID == Intrinsic::maximumnum) {
// If at least one operand is known to be positive, the result must be
// positive.
if ((KnownLHS.cannotBeOrderedLessThanZero() &&
@@ -5016,7 +5020,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
if (KnownLHS.cannotBeOrderedLessThanZero() ||
KnownRHS.cannotBeOrderedLessThanZero())
Known.knownNot(KnownFPClass::OrderedLessThanZeroMask);
- } else if (IID == Intrinsic::minnum) {
+ } else if (IID == Intrinsic::minnum || IID == Intrinsic::minimumnum) {
// If at least one operand is known to be negative, the result must be
// negative.
if ((KnownLHS.cannotBeOrderedGreaterThanZero() &&
@@ -5058,15 +5062,15 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
Known.signBitMustBeOne();
else
Known.signBitMustBeZero();
- } else if ((IID == Intrinsic::maximum || IID == Intrinsic::minimum) ||
+ } else if ((IID == Intrinsic::maximum || IID == Intrinsic::minimum || IID == Intrinsic::maximumnum || IID == Intrinsic::minimumnum) ||
((KnownLHS.isKnownNeverNegZero() ||
KnownRHS.isKnownNeverPosZero()) &&
(KnownLHS.isKnownNeverPosZero() ||
KnownRHS.isKnownNeverNegZero()))) {
- if ((IID == Intrinsic::maximum || IID == Intrinsic::maxnum) &&
+ if ((IID == Intrinsic::maximum || IID == Intrinsic::maximumnum|| IID == Intrinsic::maxnum) &&
(KnownLHS.SignBit == false || KnownRHS.SignBit == false))
Known.signBitMustBeZero();
- else if ((IID == Intrinsic::minimum || IID == Intrinsic::minnum) &&
+ else if ((IID == Intrinsic::minimum || IID == Intrinsic::minimumnum || IID == Intrinsic::minnum) &&
(KnownLHS.SignBit == true || KnownRHS.SignBit == true))
Known.signBitMustBeOne();
}
@@ -5123,7 +5127,9 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
case Intrinsic::vector_reduce_fmax:
case Intrinsic::vector_reduce_fmin:
case Intrinsic::vector_reduce_fmaximum:
- case Intrinsic::vector_reduce_fminimum: {
+ case Intrinsic::vector_reduce_fminimum:
+ case Intrinsic::vector_reduce_fmaximumnum:
+ case Intrinsic::vector_reduce_fminimumnum: {
// reduce min/max will choose an element from one of the vector elements,
// so we can infer and class information that is common to all elements.
Known = computeKnownFPClass(II->getArgOperand(0), II->getFastMathFlags(),
@@ -7131,6 +7137,8 @@ static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
case Intrinsic::maxnum:
case Intrinsic::minimum:
case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
case Intrinsic::is_fpclass:
case Intrinsic::ldexp:
case Intrinsic::frexp:
@@ -8559,6 +8567,8 @@ Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
case Intrinsic::umin: return Intrinsic::umax;
// Please note that next four intrinsics may produce the same result for
// original and inverted case even if X != Y due to NaN is handled specially.
+ case Intrinsic::maximumnum: return Intrinsic::minimumnum;
+ case Intrinsic::minimumnum: return Intrinsic::maximumnum;
case Intrinsic::maximum: return Intrinsic::minimum;
case Intrinsic::minimum: return Intrinsic::maximum;
case Intrinsic::maxnum: return Intrinsic::minnum;
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 917094267d05a..726d001ecf80f 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -78,6 +78,8 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
case Intrinsic::maxnum:
case Intrinsic::minimum:
case Intrinsic::maximum:
+ case Intrinsic::minimumnum:
+ case Intrinsic::maximumnum:
case Intrinsic::copysign:
case Intrinsic::floor:
case Intrinsic::ceil:
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index dc35f33a3a059..93138f7516c21 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -331,7 +331,11 @@ Value *CachingVPExpander::expandPredicationToFPCall(
return NewOp;
}
case Intrinsic::maxnum:
- case Intrinsic::minnum: {
+ case Intrinsic::minnum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum: {
Value *Op0 = VPI.getOperand(0);
Value *Op1 = VPI.getOperand(1);
Function *Fn = Intrinsic::getDeclaration(
@@ -498,6 +502,18 @@ CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
Reduction =
Builder.CreateBinaryIntrinsic(Intrinsic::minimum, Reduction, Start);
break;
+ case Intrinsic::vp_reduce_fmaximumnum:
+ Reduction = Builder.CreateFPMaximumReduce(RedOp);
+ transferDecorations(*Reduction, VPI);
+ Reduction =
+ Builder.CreateBinaryIntrinsic(Intrinsic::maximumnum, Reduction, Start);
+ break;
+ case Intrinsic::vp_reduce_fminimumnum:
+ Reduction = Builder.CreateFPMinimumReduce(RedOp);
+ transferDecorations(*Reduction, VPI);
+ Reduction =
+ Builder.CreateBinaryIntrinsic(Intrinsic::minimumnum, Reduction, Start);
+ break;
case Intrinsic::vp_reduce_fadd:
Reduction = Builder.CreateFAddReduce(Start, RedOp);
break;
@@ -755,6 +771,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
case Intrinsic::vp_minnum:
case Intrinsic::vp_maximum:
case Intrinsic::vp_minimum:
+ case Intrinsic::vp_maximumnum:
+ case Intrinsic::vp_minimumnum:
case Intrinsic::vp_fma:
case Intrinsic::vp_fmuladd:
return expandPredicationToFPCall(Builder, VPI,
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 547529bbe699a..ad2ef19fc67eb 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -245,6 +245,8 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc,
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FCOPYSIGN: {
// Try to constant fold these.
assert(SrcOps.size() == 2 && "Invalid sources");
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 4cc602b5c8709..54b478d3aae8d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6159,6 +6159,8 @@ bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI,
return false;
case TargetOpcode::G_FMINNUM:
case TargetOpcode::G_FMAXNUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
PropagateNaN = false;
break;
case TargetOpcode::G_FMINIMUM:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 5289b993476db..05bf06de2ace1 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1915,6 +1915,10 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_FMINIMUM;
case Intrinsic::maximum:
return TargetOpcode::G_FMAXIMUM;
+ case Intrinsic::minimumnum:
+ return TargetOpcode::G_FMINIMUMNUM;
+ case Intrinsic::maximumnum:
+ return TargetOpcode::G_FMAXIMUMNUM;
case Intrinsic::canonicalize:
return TargetOpcode::G_FCANONICALIZE;
case Intrinsic::floor:
@@ -1968,6 +1972,10 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
return TargetOpcode::G_VECREDUCE_FMINIMUM;
case Intrinsic::vector_reduce_fmaximum:
return TargetOpcode::G_VECREDUCE_FMAXIMUM;
+ case Intrinsic::vector_reduce_fminimumnum:
+ return TargetOpcode::G_VECREDUCE_FMINIMUMNUM;
+ case Intrinsic::vector_reduce_fmaximumnum:
+ return TargetOpcode::G_VECREDUCE_FMAXIMUMNUM;
case Intrinsic::vector_reduce_add:
return TargetOpcode::G_VECREDUCE_ADD;
case Intrinsic::vector_reduce_mul:
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index d8b0f52ecf9e3..e1133cd232d4f 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -465,6 +465,14 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
RTLIBCASE(FMIN_F);
case TargetOpcode::G_FMAXNUM:
RTLIBCASE(FMAX_F);
+ case TargetOpcode::G_FMINIMUM:
+ RTLIBCASE(FMINIMUM_F);
+ case TargetOpcode::G_FMAXIMUM:
+ RTLIBCASE(FMAXIMUM_F);
+ case TargetOpcode::G_FMINIMUMNUM:
+ RTLIBCASE(FMINIMUMNUM_F);
+ case TargetOpcode::G_FMAXIMUMNUM:
+ RTLIBCASE(FMAXIMUMNUM_F);
case TargetOpcode::G_FSQRT:
RTLIBCASE(SQRT_F);
case TargetOpcode::G_FRINT:
@@ -1048,6 +1056,10 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
case TargetOpcode::G_FFLOOR:
case TargetOpcode::G_FMINNUM:
case TargetOpcode::G_FMAXNUM:
+ case TargetOpcode::G_FMINIMUM:
+ case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FSQRT:
case TargetOpcode::G_FRINT:
case TargetOpcode::G_FNEARBYINT:
@@ -2887,6 +2899,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FDIV:
case TargetOpcode::G_FREM:
case TargetOpcode::G_FCEIL:
@@ -3009,7 +3023,9 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
case TargetOpcode::G_VECREDUCE_FMIN:
case TargetOpcode::G_VECREDUCE_FMAX:
case TargetOpcode::G_VECREDUCE_FMINIMUM:
- case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
+ case TargetOpcode::G_VECREDUCE_FMAXIMUM:
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM: {
if (TypeIdx != 0)
return UnableToLegalize;
Observer.changingInstr(MI);
@@ -3926,6 +3942,9 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
case G_FMINNUM:
case G_FMAXNUM:
return lowerFMinNumMaxNum(MI);
+ case G_FMINIMUMNUM:
+ case G_FMAXIMUMNUM:
+ return lowerFMinimumNumMaximumNum(MI);
case G_MERGE_VALUES:
return lowerMergeValues(MI);
case G_UNMERGE_VALUES:
@@ -4679,6 +4698,8 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
case G_FMAXNUM_IEEE:
case G_FMINIMUM:
case G_FMAXIMUM:
+ case G_FMINIMUMNUM:
+ case G_FMAXIMUMNUM:
case G_FSHL:
case G_FSHR:
case G_ROTL:
@@ -5337,8 +5358,10 @@ MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
return MIRBuilder.buildFConstant(Ty, 1.0);
case TargetOpcode::G_VECREDUCE_FMINIMUM:
case TargetOpcode::G_VECREDUCE_FMAXIMUM:
+ case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
+ case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
assert(false && "getNeutralElementForVecReduce unimplemented for "
- "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
+ "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM(_NUM)!");
}
llvm_unreachable("switch expected to return!");
}
@@ -5389,6 +5412,8 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_STRICT_FADD:
case TargetOpcode::G_STRICT_FSUB:
case TargetOpcode::G_STRICT_FMUL:
@@ -7244,6 +7269,35 @@ LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
return Legalized;
}
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerFMinimumNumMaximumNum(MachineInstr &MI) {
+ unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINIMUMNUM ?
+ TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
+
+ auto [Dst, Src0, Src1] = MI.getFirst3Regs();
+ LLT Ty = MRI.getType(Dst);
+
+ if (!MI.getFlag(MachineInstr::FmNoNans)) {
+ // Insert canonicalizes if it's possible we need to quiet to get correct
+ // sNaN behavior.
+
+ // Note this must be done here, and not as an optimization combine in the
+ // absence of a dedicate quiet-snan instruction as we're using an
+ // omni-purpose G_FCANONICALIZE.
+ if (!isKnownNeverSNaN(Src0, MRI))
+ Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
+
+ if (!isKnownNeverSNaN(Src1, MRI))
+ Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
+ }
+
+ // If there are no nans, it's safe to simply replace this with the non-IEEE
+ // version.
+ MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
+ MI.eraseFromParent();
+ return Legalized;
+}
+
LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
// Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
Register DstReg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index f455482e02943..7e4e0db118bd8 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -763,6 +763,10 @@ llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1,
return minimum(C1, C2);
case TargetOpcode::G_FMAXIMUM:
return maximum(C1, C2);
+ case TargetOpcode::G_FMINIMUMNUM:
+ return minimumnum(C1, C2);
+ case TargetOpcode::G_FMAXIMUMNUM:
+ return maximumnum(C1, C2);
case TargetOpcode::G_FMINNUM_IEEE:
case TargetOpcode::G_FMAXNUM_IEEE:
// FIXME: These operations were unfortunately named. fminnum/fmaxnum do not
@@ -858,6 +862,18 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
return isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI, SNaN) ||
isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI, SNaN);
}
+ case TargetOpcode::G_FMINIMUM:
+ case TargetOpcode::G_FMAXIMUM: {
+ return isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI) &&
+ isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI);
+ }
+ case TargetOpcode::G_FMINIMUMNUM:
+ case TargetOpcode::G_FMAXIMUMNUM: {
+ // Only one needs to be known not-nan, since it will be returned if the
+ // other ends up being one.
+ return isKnownNeverNaN(DefMI->getOperand(1).getReg(), MRI) ||
+ isKnownNeverNaN(DefMI->getOperand(2).getReg(), MRI);
+ }
}
if (SNaN) {
@@ -1699,9 +1715,11 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
case TargetOpcode::G_FMA:
case TargetOpcode::G_FMAD:
case TargetOpcode::G_FMAXIMUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
case TargetOpcode::G_FMAXNUM:
case TargetOpcode::G_FMAXNUM_IEEE:
case TargetOpcode::G_FMINIMUM:
+ case TargetOpcode::G_FMINIMUMNUM:
case TargetOpcode::G_FMINNUM:
case TargetOpcode::G_FMINNUM_IEEE:
case TargetOpcode::G_FMUL:
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8607b50175359..1504495a9131a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1939,7 +1939,9 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::FMINNUM:
case ISD::FMAXNUM:
case ISD::FMINIMUM:
- case ISD::FMAXIMUM: return visitFMinMax(N);
+ case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM: return visitFMinMax(N);
case ISD::FCEIL: return visitFCEIL(N);
case ISD::FTRUNC: return visitFTRUNC(N);
case ISD::FFREXP: return visitFFREXP(N);
@@ -1982,7 +1984,9 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
- case ISD::VECREDUCE_FMINIMUM: return visitVECREDUCE(N);
+ case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM: return visitVECREDUCE(N);
#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
#include "llvm/IR/VPIntrinsics.def"
return visitVPOp(N);
@@ -6068,6 +6072,7 @@ static bool arebothOperandsNotNan(SDValue Operand1, SDValue Operand2,
return DAG.isKnownNeverNaN(Operand2) && DAG.isKnownNeverNaN(Operand1);
}
+// FIXME: use FMINIMUMNUM if possible, such as for RISC-V.
static unsigned getMinMaxOpcodeForFP(SDValue Operand1, SDValue Operand2,
ISD::CondCode CC, unsigned OrAndOpcode,
SelectionDAG &DAG,
@@ -18000,7 +18005,7 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) {
const SDNodeFlags Flags = N->getFlags();
unsigned Opc = N->getOpcode();
bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
- bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
+ bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM;
SelectionDAG::FlagInserter FlagsInserter(DAG, N);
// Constant fold.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index bfc2273c9425c..dbfce60ce607d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1203,6 +1203,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
case ISD::IS_FPCLASS:
Action = TLI.getOperationAction(
Node->getOpcode(), Node->getOperand(0).getValueType());
@@ -1224,6 +1226,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
case ISD::VP_REDUCE_FMIN:
case ISD::VP_REDUCE_FMAXIMUM:
case ISD::VP_REDUCE_FMINIMUM:
+ case ISD::VP_REDUCE_FMAXIMUMNUM:
+ case ISD::VP_REDUCE_FMINIMUMNUM:
case ISD::VP_REDUCE_SEQ_FADD:
case ISD::VP_REDUCE_SEQ_FMUL:
Action = TLI.getOperationAction(
@@ -3621,6 +3625,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(Expanded);
break;
}
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM: {
+ if (SDValue Expanded = TLI.expandFMINIMUMNUM_FMAXIMUMNUM(Node, DAG))
+ Results.push_back(Expanded);
+ break;
+ }
case ISD::FSIN:
case ISD::FCOS: {
EVT VT = Node->getValueType(0);
@@ -4291,6 +4301,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
Results.push_back(TLI.expandVecReduce(Node, DAG));
break;
case ISD::VP_CTTZ_ELTS:
@@ -4460,15 +4472,36 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::FMIN_F80, RTLIB::FMIN_F128,
RTLIB::FMIN_PPCF128, Results);
break;
- // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
- // libcall legalization for these nodes, but there is no default expasion for
- // these nodes either (see PR63267 for example).
case ISD::FMAXNUM:
case ISD::STRICT_FMAXNUM:
ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
RTLIB::FMAX_F80, RTLIB::FMAX_F128,
RTLIB::FMAX_PPCF128, Results);
break;
+ case ISD::FMINIMUM:
+ case ISD::STRICT_FMINIMUM:
+ ExpandFPLibCall(Node, RTLIB::FMINIMUM_F32, RTLIB::FMINIMUM_F64,
+ RTLIB::FMINIMUM_F80, RTLIB::FMINIMUM_F128,
+ RTLIB::FMINIMUM_PPCF128, Results);
+ break;
+ case ISD::FMAXIMUM:
+ case ISD::STRICT_FMAXIMUM:
+ ExpandFPLibCall(Node, RTLIB::FMAXIMUM_F32, RTLIB::FMAXIMUM_F64,
+ RTLIB::FMAXIMUM_F80, RTLIB::FMAXIMUM_F128,
+ RTLIB::FMAXIMUM_PPCF128, Results);
+ break;
+ case ISD::FMINIMUMNUM:
+ case ISD::STRICT_FMINIMUMNUM:
+ ExpandFPLibCall(Node, RTLIB::FMINIMUMNUM_F32, RTLIB::FMINIMUMNUM_F64,
+ RTLIB::FMINIMUMNUM_F80, RTLIB::FMINIMUMNUM_F128,
+ RTLIB::FMINIMUMNUM_PPCF128, Results);
+ break;
+ case ISD::FMAXIMUMNUM:
+ case ISD::STRICT_FMAXIMUMNUM:
+ ExpandFPLibCall(Node, RTLIB::FMAXIMUMNUM_F32, RTLIB::FMAXIMUMNUM_F64,
+ RTLIB::FMAXIMUMNUM_F80, RTLIB::FMAXIMUMNUM_F128,
+ RTLIB::FMAXIMUMNUM_PPCF128, Results);
+ break;
case ISD::FSQRT:
case ISD::STRICT_FSQRT:
ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
@@ -5026,6 +5059,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
+ Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUMNUM ||
+ Node->getOpcode() == ISD::VP_REDUCE_FMINIMUMNUM ||
Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
OVT = Node->getOperand(1).getSimpleValueType();
if (Node->getOpcode() == ISD::BR_CC ||
@@ -5341,6 +5376,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::FMAXNUM:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::FPOW:
Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
@@ -5356,6 +5393,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::STRICT_FDIV:
case ISD::STRICT_FMINNUM:
case ISD::STRICT_FMAXNUM:
+ case ISD::STRICT_FMINIMUM:
+ case ISD::STRICT_FMAXIMUM:
+ case ISD::STRICT_FMINIMUMNUM:
+ case ISD::STRICT_FMAXIMUMNUM:
case ISD::STRICT_FREM:
case ISD::STRICT_FPOW:
Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
@@ -5702,6 +5743,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
case ISD::VP_REDUCE_FMIN:
case ISD::VP_REDUCE_FMAXIMUM:
case ISD::VP_REDUCE_FMINIMUM:
+ case ISD::VP_REDUCE_FMAXIMUMNUM:
+ case ISD::VP_REDUCE_FMINIMUMNUM:
case ISD::VP_REDUCE_SEQ_FADD:
Results.push_back(PromoteReduction(Node));
break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index fc96ecdc66280..0772e33c4ff33 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -74,6 +74,14 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
case ISD::STRICT_FMAXNUM:
case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
+ case ISD::STRICT_FMINIMUM:
+ case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
+ case ISD::STRICT_FMAXIMUM:
+ case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
+ case ISD::STRICT_FMINIMUMNUM:
+ case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
+ case ISD::STRICT_FMAXIMUMNUM:
+ case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
case ISD::STRICT_FADD:
case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
@@ -150,7 +158,9 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMAXIMUM:
- case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
+ case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM: R = SoftenFloatRes_VECREDUCE(N); break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
// clang-format on
@@ -309,6 +319,50 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
RTLIB::FMAX_PPCF128));
}
+SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
+ if (SDValue SelCC = TLI.createSelectForFMINIMUM_FMAXIMUM(N, DAG))
+ return SoftenFloatRes_SELECT_CC(SelCC.getNode());
+ return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMINIMUM_F32,
+ RTLIB::FMINIMUM_F64,
+ RTLIB::FMINIMUM_F80,
+ RTLIB::FMINIMUM_F128,
+ RTLIB::FMINIMUM_PPCF128));
+}
+
+SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
+ if (SDValue SelCC = TLI.createSelectForFMINIMUM_FMAXIMUM(N, DAG))
+ return SoftenFloatRes_SELECT_CC(SelCC.getNode());
+ return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMAXIMUM_F32,
+ RTLIB::FMAXIMUM_F64,
+ RTLIB::FMAXIMUM_F80,
+ RTLIB::FMAXIMUM_F128,
+ RTLIB::FMAXIMUM_PPCF128));
+}
+
+SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
+ if (SDValue SelCC = TLI.createSelectForFMINIMUMNUM_FMAXIMUMNUM(N, DAG))
+ return SoftenFloatRes_SELECT_CC(SelCC.getNode());
+ return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMINIMUMNUM_F32,
+ RTLIB::FMINIMUMNUM_F64,
+ RTLIB::FMINIMUMNUM_F80,
+ RTLIB::FMINIMUMNUM_F128,
+ RTLIB::FMINIMUMNUM_PPCF128));
+}
+
+SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
+ if (SDValue SelCC = TLI.createSelectForFMINIMUMNUM_FMAXIMUMNUM(N, DAG))
+ return SoftenFloatRes_SELECT_CC(SelCC.getNode());
+ return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMAXIMUMNUM_F32,
+ RTLIB::FMAXIMUMNUM_F64,
+ RTLIB::FMAXIMUMNUM_F80,
+ RTLIB::FMAXIMUMNUM_F128,
+ RTLIB::FMAXIMUMNUM_PPCF128));
+}
+
SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
RTLIB::ADD_F32,
@@ -1348,6 +1402,14 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
case ISD::STRICT_FMAXNUM:
case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
+ case ISD::STRICT_FMINIMUM:
+ case ISD::FMINIMUM: ExpandFloatRes_FMINIMUM(N, Lo, Hi); break;
+ case ISD::STRICT_FMAXIMUM:
+ case ISD::FMAXIMUM: ExpandFloatRes_FMAXIMUM(N, Lo, Hi); break;
+ case ISD::STRICT_FMINIMUMNUM:
+ case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
+ case ISD::STRICT_FMAXIMUMNUM:
+ case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
case ISD::STRICT_FADD:
case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
@@ -1490,6 +1552,38 @@ void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
RTLIB::FMAX_PPCF128), Lo, Hi);
}
+void DAGTypeLegalizer::ExpandFloatRes_FMINIMUM(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMINIMUM_F32, RTLIB::FMINIMUM_F64,
+ RTLIB::FMINIMUM_F80, RTLIB::FMINIMUM_F128,
+ RTLIB::FMINIMUM_PPCF128), Lo, Hi);
+}
+
+void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUM(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMAXIMUM_F32, RTLIB::FMAXIMUM_F64,
+ RTLIB::FMAXIMUM_F80, RTLIB::FMAXIMUM_F128,
+ RTLIB::FMAXIMUM_PPCF128), Lo, Hi);
+}
+
+void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMINIMUMNUM_F32, RTLIB::FMINIMUMNUM_F64,
+ RTLIB::FMINIMUMNUM_F80, RTLIB::FMINIMUMNUM_F128,
+ RTLIB::FMINIMUMNUM_PPCF128), Lo, Hi);
+}
+
+void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
+ RTLIB::FMAXIMUMNUM_F32, RTLIB::FMAXIMUMNUM_F64,
+ RTLIB::FMAXIMUMNUM_F80, RTLIB::FMAXIMUMNUM_F128,
+ RTLIB::FMAXIMUMNUM_PPCF128), Lo, Hi);
+}
+
void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
SDValue &Hi) {
ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
@@ -2486,6 +2580,8 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FDIV:
case ISD::FMAXIMUM:
case ISD::FMINIMUM:
+ case ISD::FMAXIMUMNUM:
+ case ISD::FMINIMUMNUM:
case ISD::FMAXNUM:
case ISD::FMINNUM:
case ISD::FMUL:
@@ -2521,6 +2617,8 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
R = PromoteFloatRes_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
@@ -2919,6 +3017,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::FDIV:
case ISD::FMAXIMUM:
case ISD::FMINIMUM:
+ case ISD::FMAXIMUMNUM:
+ case ISD::FMINIMUMNUM:
case ISD::FMAXNUM:
case ISD::FMINNUM:
case ISD::FMUL:
@@ -2950,6 +3050,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_FMAX:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
R = SoftPromoteHalfRes_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index d925089d5689f..2472595ff5855 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -535,6 +535,10 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_FABS(SDNode *N);
SDValue SoftenFloatRes_FMINNUM(SDNode *N);
SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
+ SDValue SoftenFloatRes_FMINIMUM(SDNode *N);
+ SDValue SoftenFloatRes_FMAXIMUM(SDNode *N);
+ SDValue SoftenFloatRes_FMINIMUMNUM(SDNode *N);
+ SDValue SoftenFloatRes_FMAXIMUMNUM(SDNode *N);
SDValue SoftenFloatRes_FADD(SDNode *N);
SDValue SoftenFloatRes_FCBRT(SDNode *N);
SDValue SoftenFloatRes_FCEIL(SDNode *N);
@@ -619,6 +623,10 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandFloatRes_FMINIMUM (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandFloatRes_FMAXIMUM (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandFloatRes_FMINIMUMNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandFloatRes_FMAXIMUMNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 6acbc044d6731..c5f2e34d16aa1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -393,6 +393,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::FMAXNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::FCOPYSIGN:
case ISD::FSQRT:
case ISD::FSIN:
@@ -478,6 +480,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(0).getValueType());
break;
@@ -995,6 +999,13 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
return;
}
break;
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
+ if (SDValue Expanded = TLI.expandFMINIMUMNUM_FMAXIMUMNUM(Node, DAG)) {
+ Results.push_back(Expanded);
+ return;
+ }
+ break;
case ISD::SMIN:
case ISD::SMAX:
case ISD::UMIN:
@@ -1083,6 +1094,8 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
Results.push_back(TLI.expandVecReduce(Node, DAG));
return;
case ISD::VECREDUCE_SEQ_FADD:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index ec05135915664..9353d6b921e06 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -135,6 +135,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::FMAXNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::FLDEXP:
case ISD::SMIN:
case ISD::SMAX:
@@ -1176,10 +1178,10 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::FMUL: case ISD::VP_FMUL:
case ISD::FMINNUM: case ISD::VP_FMINNUM:
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
- case ISD::FMINIMUM:
- case ISD::VP_FMINIMUM:
- case ISD::FMAXIMUM:
- case ISD::VP_FMAXIMUM:
+ case ISD::FMINIMUM: case ISD::FMAXIMUM:
+ case ISD::VP_FMINIMUM: case ISD::VP_FMAXIMUM:
+ case ISD::FMINIMUMNUM: case ISD::FMAXIMUMNUM:
+ case ISD::VP_FMINIMUMNUM: case ISD::VP_FMAXIMUMNUM:
case ISD::SDIV: case ISD::VP_SDIV:
case ISD::UDIV: case ISD::VP_UDIV:
case ISD::FDIV: case ISD::VP_FDIV:
@@ -3115,6 +3117,8 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMAXIMUM:
case ISD::VECREDUCE_FMINIMUM:
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VECREDUCE_FMINIMUMNUM:
Res = SplitVecOp_VECREDUCE(N, OpNo);
break;
case ISD::VECREDUCE_SEQ_FADD:
@@ -3138,6 +3142,8 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VP_REDUCE_FMIN:
case ISD::VP_REDUCE_FMAXIMUM:
case ISD::VP_REDUCE_FMINIMUM:
+ case ISD::VP_REDUCE_FMAXIMUMNUM:
+ case ISD::VP_REDUCE_FMINIMUMNUM:
Res = SplitVecOp_VP_REDUCE(N, OpNo);
break;
case ISD::VP_CTTZ_ELTS:
@@ -4239,10 +4245,10 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SRL: case ISD::VP_LSHR:
case ISD::FMINNUM: case ISD::VP_FMINNUM:
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
- case ISD::FMINIMUM:
- case ISD::VP_FMINIMUM:
- case ISD::FMAXIMUM:
- case ISD::VP_FMAXIMUM:
+ case ISD::FMINIMUM: case ISD::FMAXIMUM:
+ case ISD::VP_FMINIMUM: case ISD::VP_FMAXIMUM:
+ case ISD::FMINIMUMNUM: case ISD::FMAXIMUMNUM:
+ case ISD::VP_FMINIMUMNUM: case ISD::VP_FMAXIMUMNUM:
case ISD::SMIN: case ISD::VP_SMIN:
case ISD::SMAX: case ISD::VP_SMAX:
case ISD::UMIN: case ISD::VP_UMIN:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index b05649c6ce955..884c355380add 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -475,6 +475,12 @@ ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
case ISD::VECREDUCE_FMINIMUM:
case ISD::VP_REDUCE_FMINIMUM:
return ISD::FMINIMUM;
+ case ISD::VECREDUCE_FMAXIMUMNUM:
+ case ISD::VP_REDUCE_FMAXIMUMNUM:
+ return ISD::FMAXIMUMNUM;
+ case ISD::VECREDUCE_FMINIMUMNUM:
+ case ISD::VP_REDUCE_FMINIMUMNUM:
+ return ISD::FMINIMUMNUM;
}
}
@@ -5422,9 +5428,13 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
}
case ISD::FMINIMUM:
case ISD::FMAXIMUM: {
- // TODO: Does this quiet or return the origina NaN as-is?
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
- isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
+ isKnownNeverNaN(Op.getOperand(1), false, Depth + 1);
+ }
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM: {
+ return isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) ||
+ isKnownNeverNaN(Op.getOperand(1), false, Depth + 1);
}
case ISD::EXTRACT_VECTOR_ELT: {
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
@@ -6726,6 +6736,10 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
return getConstantFP(minimum(C1, C2), DL, VT);
case ISD::FMAXIMUM:
return getConstantFP(maximum(C1, C2), DL, VT);
+ case ISD::FMINIMUMNUM:
+ return getConstantFP(minimumnum(C1, C2), DL, VT);
+ case ISD::FMAXIMUMNUM:
+ return getConstantFP(maximumnum(C1, C2), DL, VT);
default: break;
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ca352da5d36eb..9da66140c4fd0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6826,6 +6826,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
getValue(I.getArgOperand(0)),
getValue(I.getArgOperand(1)), Flags));
return;
+ case Intrinsic::minimumnum:
+ setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,
+ getValue(I.getArgOperand(0)).getValueType(),
+ getValue(I.getArgOperand(0)),
+ getValue(I.getArgOperand(1)), Flags));
+ return;
+ case Intrinsic::maximumnum:
+ setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,
+ getValue(I.getArgOperand(0)).getValueType(),
+ getValue(I.getArgOperand(0)),
+ getValue(I.getArgOperand(1)), Flags));
+ return;
case Intrinsic::copysign:
setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
getValue(I.getArgOperand(0)).getValueType(),
@@ -9130,6 +9142,26 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
if (visitBinaryFloatCall(I, ISD::FMAXNUM))
return;
break;
+ case LibFunc_fminimum:
+ case LibFunc_fminimumf:
+ if (visitBinaryFloatCall(I, ISD::FMINIMUM))
+ return;
+ break;
+ case LibFunc_fmaximum:
+ case LibFunc_fmaximumf:
+ if (visitBinaryFloatCall(I, ISD::FMAXIMUM))
+ return;
+ break;
+ case LibFunc_fminimum_num:
+ case LibFunc_fminimum_numf:
+ if (visitBinaryFloatCall(I, ISD::FMINIMUMNUM))
+ return;
+ break;
+ case LibFunc_fmaximum_num:
+ case LibFunc_fmaximum_numf:
+ if (visitBinaryFloatCall(I, ISD::FMAXIMUMNUM))
+ return;
+ break;
case LibFunc_sin:
case LibFunc_sinf:
case LibFunc_sinl:
@@ -10554,6 +10586,12 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
case Intrinsic::vector_reduce_fminimum:
Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
break;
+ case Intrinsic::vector_reduce_fmaximumnum:
+ Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUMNUM, dl, VT, Op1, SDFlags);
+ break;
+ case Intrinsic::vector_reduce_fminimumnum:
+ Res = DAG.getNode(ISD::VECREDUCE_FMINIMUMNUM, dl, VT, Op1, SDFlags);
+ break;
default:
llvm_unreachable("Unhandled vector reduce intrinsic");
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 59742e90c6791..5c33d6c7f5f2f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -201,6 +201,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::STRICT_FMINIMUM: return "strict_fminimum";
case ISD::FMAXIMUM: return "fmaximum";
case ISD::STRICT_FMAXIMUM: return "strict_fmaximum";
+ case ISD::FMINIMUMNUM: return "fminimumnum";
+ case ISD::STRICT_FMINIMUMNUM: return "strict_fminimumnum";
+ case ISD::FMAXIMUMNUM: return "fmaximumnum";
+ case ISD::STRICT_FMAXIMUMNUM: return "strict_fmaximumnum";
case ISD::FNEG: return "fneg";
case ISD::FSQRT: return "fsqrt";
case ISD::STRICT_FSQRT: return "strict_fsqrt";
@@ -524,6 +528,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::VECREDUCE_FMIN: return "vecreduce_fmin";
case ISD::VECREDUCE_FMAXIMUM: return "vecreduce_fmaximum";
case ISD::VECREDUCE_FMINIMUM: return "vecreduce_fminimum";
+ case ISD::VECREDUCE_FMAXIMUMNUM: return "vecreduce_fmaximumnum";
+ case ISD::VECREDUCE_FMINIMUMNUM: return "vecreduce_fminimumnum";
case ISD::STACKMAP:
return "stackmap";
case ISD::PATCHPOINT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 37c72339fe295..c3b1bf0f4e4a2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8377,25 +8377,8 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
report_fatal_error(
"Expanding fminnum/fmaxnum for scalable vectors is undefined.");
- if (isOperationLegalOrCustom(NewOp, VT)) {
- SDValue Quiet0 = Node->getOperand(0);
- SDValue Quiet1 = Node->getOperand(1);
-
- if (!Node->getFlags().hasNoNaNs()) {
- // Insert canonicalizes if it's possible we need to quiet to get correct
- // sNaN behavior.
- if (!DAG.isKnownNeverSNaN(Quiet0)) {
- Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
- Node->getFlags());
- }
- if (!DAG.isKnownNeverSNaN(Quiet1)) {
- Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
- Node->getFlags());
- }
- }
-
- return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
- }
+ if (isOperationLegalOrCustom(NewOp, VT))
+ return DAG.getNode(NewOp, dl, VT, Node->getOperand(0), Node->getOperand(1), Node->getFlags());
// If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
// instead if there are no NaNs and there can't be an incompatible zero
@@ -8419,6 +8402,30 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
return SDValue();
}
+SDValue
+TargetLowering::createSelectForFMINIMUM_FMAXIMUM(SDNode *Node,
+ SelectionDAG &DAG) const {
+ unsigned Opcode = Node->getOpcode();
+ assert((Opcode == ISD::FMINIMUM || Opcode == ISD::FMAXIMUM ||
+ Opcode == ISD::STRICT_FMINIMUM || Opcode == ISD::STRICT_FMAXIMUM) &&
+ "Wrong opcode");
+
+ if (Node->getFlags().hasNoNaNs()) {
+ ISD::CondCode Pred = Opcode == ISD::FMINIMUM ? ISD::SETLT : ISD::SETGT;
+ SDValue Op1 = Node->getOperand(0);
+ SDValue Op2 = Node->getOperand(1);
+ SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred);
+ // Copy FMF flags, but always set the no-signed-zeros flag
+ // as this is implied by the FMINIMUM/FMAXIMUM semantics.
+ SDNodeFlags Flags = Node->getFlags();
+ Flags.setNoSignedZeros(true);
+ SelCC->setFlags(Flags);
+ return SelCC;
+ }
+
+ return SDValue();
+}
+
SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
SelectionDAG &DAG) const {
SDLoc DL(N);
@@ -8483,6 +8490,83 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
return MinMax;
}
+SDValue
+TargetLowering::createSelectForFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
+ SelectionDAG &DAG) const {
+ unsigned Opcode = Node->getOpcode();
+ assert((Opcode == ISD::FMINIMUMNUM || Opcode == ISD::FMAXIMUMNUM ||
+ Opcode == ISD::STRICT_FMINIMUMNUM || Opcode == ISD::STRICT_FMAXIMUMNUM) &&
+ "Wrong opcode");
+
+ if (Node->getFlags().hasNoNaNs()) {
+ ISD::CondCode Pred = Opcode == ISD::FMINIMUMNUM ? ISD::SETLT : ISD::SETGT;
+ SDValue Op1 = Node->getOperand(0);
+ SDValue Op2 = Node->getOperand(1);
+ SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred);
+ // Copy FMF flags, but always set the no-signed-zeros flag
+ // as this is implied by the FMINNUM/FMAXNUM semantics.
+ SDNodeFlags Flags = Node->getFlags();
+ Flags.setNoSignedZeros(true);
+ SelCC->setFlags(Flags);
+ return SelCC;
+ }
+
+ return SDValue();
+}
+
+SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ unsigned NewOp = Node->getOpcode() == ISD::FMINIMUMNUM ?
+ ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
+ EVT VT = Node->getValueType(0);
+
+ if (VT.isScalableVector())
+ report_fatal_error(
+ "Expanding fminimumnum/fmaximumnum for scalable vectors is undefined.");
+
+ if (isOperationLegalOrCustom(NewOp, VT)) {
+ SDValue Quiet0 = Node->getOperand(0);
+ SDValue Quiet1 = Node->getOperand(1);
+
+ if (!Node->getFlags().hasNoNaNs()) {
+ // Insert canonicalizes if it's possible we need to quiet to get correct
+ // sNaN behavior.
+ if (!DAG.isKnownNeverSNaN(Quiet0)) {
+ Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
+ Node->getFlags());
+ }
+ if (!DAG.isKnownNeverSNaN(Quiet1)) {
+ Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
+ Node->getFlags());
+ }
+ }
+
+ return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
+ }
+
+ // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
+ // instead if there are no NaNs and there can't be an incompatible zero
+ // compare: at least one operand isn't +/-0, or there are no signed-zeros.
+ if ((Node->getFlags().hasNoNaNs() ||
+ (DAG.isKnownNeverNaN(Node->getOperand(0)) &&
+ DAG.isKnownNeverNaN(Node->getOperand(1)))) &&
+ (Node->getFlags().hasNoSignedZeros() ||
+ DAG.isKnownNeverZeroFloat(Node->getOperand(0)) ||
+ DAG.isKnownNeverZeroFloat(Node->getOperand(1)))) {
+ unsigned IEEE2018Op =
+ Node->getOpcode() == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
+ if (isOperationLegalOrCustom(IEEE2018Op, VT))
+ return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
+ Node->getOperand(1), Node->getFlags());
+ }
+
+ if (SDValue SelCC = createSelectForFMINIMUMNUM_FMAXIMUMNUM(Node, DAG))
+ return SelCC;
+
+ return SDValue();
+}
+
/// Returns a true value if if this FPClassTest can be performed with an ordered
/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
/// std::nullopt if it cannot be performed as a compare with 0.
@@ -11032,13 +11116,29 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
// of comparisons and selects.
bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
isOperationLegal(ISD::FMAXNUM, SrcVT);
- if (AreExactFloatBounds && MinMaxLegal) {
+ bool MinMaxIEEELegal = isOperationLegal(ISD::FMINNUM_IEEE, SrcVT) &&
+ isOperationLegal(ISD::FMAXNUM_IEEE, SrcVT);
+ bool MinMaxNumLegal = isOperationLegal(ISD::FMINIMUMNUM, SrcVT) &&
+ isOperationLegal(ISD::FMAXIMUMNUM, SrcVT);
+ bool CanonLegal = isOperationLegal(ISD::FCANONICALIZE, SrcVT);
+ if (AreExactFloatBounds && (MinMaxNumLegal || ((MinMaxIEEELegal || MinMaxLegal) && CanonLegal))) {
SDValue Clamped = Src;
+ if (MinMaxNumLegal) {
+ Clamped = DAG.getNode(ISD::FMAXIMUMNUM, dl, SrcVT, Clamped, MinFloatNode);
+ Clamped = DAG.getNode(ISD::FMINIMUMNUM, dl, SrcVT, Clamped, MaxFloatNode);
+ } else if (MinMaxIEEELegal && CanonLegal) {
+ Clamped = DAG.getNode(ISD::FCANONICALIZE, dl, SrcVT, Clamped);
+ Clamped = DAG.getNode(ISD::FMAXNUM_IEEE, dl, SrcVT, Clamped, MinFloatNode);
+ Clamped = DAG.getNode(ISD::FMINNUM_IEEE, dl, SrcVT, Clamped, MaxFloatNode);
+ } else if (MinMaxLegal && CanonLegal) {
// Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
// Clamp by MaxFloat from above. NaN cannot occur.
Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
+ }else{
+ llvm_unreachable("No usable fmax/fmin instructions!");
+ }
// Convert clamped value to integer.
SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
dl, DstVT, Clamped);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 82a59918b085b..fd8e476471a81 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -908,6 +908,7 @@ void TargetLoweringBase::initActions() {
ISD::FMINNUM, ISD::FMAXNUM,
ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
ISD::FMINIMUM, ISD::FMAXIMUM,
+ ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
ISD::FMAD, ISD::SMIN,
ISD::SMAX, ISD::UMIN,
ISD::UMAX, ISD::ABS,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index df17767138d90..b8af1404ebde0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -811,7 +811,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto Op :
{ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
ISD::FRINT, ISD::FTRUNC, ISD::FROUND,
- ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM,
+ ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
ISD::FMINIMUM, ISD::FMAXIMUM, ISD::LROUND,
ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FNEARBYINT,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 4830033b23527..1a55bca35ace1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4831,6 +4831,14 @@ def : Pat<(v1f64 (fmaxnum (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
(FMAXNMDrr FPR64:$Rn, FPR64:$Rm)>;
def : Pat<(v1f64 (fminnum (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
(FMINNMDrr FPR64:$Rn, FPR64:$Rm)>;
+def : Pat<(fminnum_ieee (f64 FPR64:$a), (f64 FPR64:$b)),
+ (FMINNMDrr FPR64:$a, FPR64:$b)>;
+def : Pat<(fminnum_ieee (f32 FPR32:$a), (f32 FPR32:$b)),
+ (FMINNMSrr FPR32:$a, FPR32:$b)>;
+def : Pat<(f32 (fcanonicalize f32:$a)),
+ (FMINSrr f32:$a, f32:$a)>;
+def : Pat<(f64 (fcanonicalize f64:$a)),
+ (FMINDrr f64:$a, f64:$a)>;
//===----------------------------------------------------------------------===//
// Floating point three operand instructions.
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 41462cceef51d..3874d5cb32f39 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1642,7 +1642,7 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,
ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,
ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,
- ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS, ISD::FLDEXP,
+ ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FSINCOS, ISD::FLDEXP,
// Misc:
ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,
// Vector:
@@ -1776,6 +1776,8 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
+ setOperationAction(ISD::FMINIMUMNUM, MVT::f32, Legal);
+ setOperationAction(ISD::FMAXIMUMNUM, MVT::f32, Legal);
setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
@@ -1825,6 +1827,8 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
if (Subtarget.hasV67Ops()) {
setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
+ setOperationAction(ISD::FMINIMUMNUM, MVT::f64, Legal);
+ setOperationAction(ISD::FMAXIMUMNUM, MVT::f64, Legal);
setOperationAction(ISD::FMUL, MVT::f64, Legal);
}
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 81035849491bc..fc3f01b32216d 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -129,6 +129,8 @@ HexagonTargetLowering::initializeHVXLowering() {
setOperationAction(ISD::FMUL, T, Legal);
setOperationAction(ISD::FMINNUM, T, Legal);
setOperationAction(ISD::FMAXNUM, T, Legal);
+ setOperationAction(ISD::FMINIMUMNUM, T, Legal);
+ setOperationAction(ISD::FMAXIMUMNUM, T, Legal);
setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom);
@@ -166,6 +168,8 @@ HexagonTargetLowering::initializeHVXLowering() {
setOperationAction(ISD::FMUL, P, Custom);
setOperationAction(ISD::FMINNUM, P, Custom);
setOperationAction(ISD::FMAXNUM, P, Custom);
+ setOperationAction(ISD::FMINIMUMNUM, P, Custom);
+ setOperationAction(ISD::FMAXIMUMNUM, P, Custom);
setOperationAction(ISD::SETCC, P, Custom);
setOperationAction(ISD::VSELECT, P, Custom);
@@ -3168,6 +3172,8 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::FMUL:
case ISD::FMINNUM:
case ISD::FMAXNUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::MULHS:
case ISD::MULHU:
case ISD::AND:
diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td
index ea7c4acd0e830..ae0f34c5f280d 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatterns.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td
@@ -1563,6 +1563,8 @@ def: OpR_RR_pat<F2_sfsub, pf2<fsub>, f32, F32>;
def: OpR_RR_pat<F2_sfmpy, pf2<fmul>, f32, F32>;
def: OpR_RR_pat<F2_sfmin, pf2<fminnum>, f32, F32>;
def: OpR_RR_pat<F2_sfmax, pf2<fmaxnum>, f32, F32>;
+def: OpR_RR_pat<F2_sfmin, pf2<fminimumnum>, f32, F32>;
+def: OpR_RR_pat<F2_sfmax, pf2<fmaximumnum>, f32, F32>;
let Predicates = [HasV66] in {
def: OpR_RR_pat<F2_dfadd, pf2<fadd>, f64, F64>;
@@ -1584,6 +1586,8 @@ let Predicates = [HasV67,UseUnsafeMath], AddedComplexity = 50 in {
let Predicates = [HasV67] in {
def: OpR_RR_pat<F2_dfmin, pf2<fminnum>, f64, F64>;
def: OpR_RR_pat<F2_dfmax, pf2<fmaxnum>, f64, F64>;
+ def: OpR_RR_pat<F2_dfmin, pf2<fminimumnum>, f64, F64>;
+ def: OpR_RR_pat<F2_dfmax, pf2<fmaximumnum>, f64, F64>;
def: Pat<(fmul F64:$Rs, F64:$Rt), (DfMpy (F2_dfmpyfix $Rs, $Rt),
(F2_dfmpyfix $Rt, $Rs))>;
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index 5b8386416a5f0..32e7c65bf4db4 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -512,6 +512,10 @@ let Predicates = [UseHVXV68, UseHVX128B, UseHVXQFloat] in {
def: OpR_RR_pat<V6_vmax_hf, pf2<fmaxnum>, VecF16, HVF16>;
def: OpR_RR_pat<V6_vmin_sf, pf2<fminnum>, VecF32, HVF32>;
def: OpR_RR_pat<V6_vmax_sf, pf2<fmaxnum>, VecF32, HVF32>;
+ def: OpR_RR_pat<V6_vmin_hf, pf2<fminimumnum>, VecF16, HVF16>;
+ def: OpR_RR_pat<V6_vmax_hf, pf2<fmaximumnum>, VecF16, HVF16>;
+ def: OpR_RR_pat<V6_vmin_sf, pf2<fminimumnum>, VecF32, HVF32>;
+ def: OpR_RR_pat<V6_vmax_sf, pf2<fmaximumnum>, VecF32, HVF32>;
}
let Predicates = [UseHVXV68, UseHVX128B, UseHVXIEEEFP] in {
@@ -525,6 +529,10 @@ let Predicates = [UseHVXV68, UseHVX128B, UseHVXIEEEFP] in {
def: OpR_RR_pat<V6_vfmax_hf, pf2<fmaxnum>, VecF16, HVF16>;
def: OpR_RR_pat<V6_vfmin_sf, pf2<fminnum>, VecF32, HVF32>;
def: OpR_RR_pat<V6_vfmax_sf, pf2<fmaxnum>, VecF32, HVF32>;
+ def: OpR_RR_pat<V6_vfmin_hf, pf2<fminimumnum>, VecF16, HVF16>;
+ def: OpR_RR_pat<V6_vfmax_hf, pf2<fmaximumnum>, VecF16, HVF16>;
+ def: OpR_RR_pat<V6_vfmin_sf, pf2<fminimumnum>, VecF32, HVF32>;
+ def: OpR_RR_pat<V6_vfmax_sf, pf2<fmaximumnum>, VecF32, HVF32>;
}
let Predicates = [UseHVX] in {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index c6d11b8a8bd7e..951580cc94811 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -372,7 +372,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
// FP Operations
getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FNEG,
- G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM})
+ G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM, G_FMINIMUMNUM})
.legalIf(typeIsScalarFPArith(0, ST));
getActionDefinitionsBuilder(G_FCOPYSIGN)
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index 686c8d89a7321..4ad5808fb075f 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -281,7 +281,9 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
case TargetOpcode::G_FNEG:
case TargetOpcode::G_FSQRT:
case TargetOpcode::G_FMAXNUM:
- case TargetOpcode::G_FMINNUM: {
+ case TargetOpcode::G_FMINNUM:
+ case TargetOpcode::G_FMAXIMUMNUM:
+ case TargetOpcode::G_FMINIMUMNUM: {
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
TypeSize Size = Ty.getSizeInBits();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 05859a1f4898b..0097d2a6ab709 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -431,7 +431,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
}
static const unsigned FPLegalNodeTypes[] = {
- ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT,
+ ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::LRINT,
ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
ISD::STRICT_LRINT, ISD::STRICT_LLRINT, ISD::STRICT_LROUND,
ISD::STRICT_LLROUND, ISD::STRICT_FMA, ISD::STRICT_FADD,
@@ -455,7 +455,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BITCAST, MVT::i16, Custom);
static const unsigned ZfhminZfbfminPromoteOps[] = {
- ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD,
+ ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FADD,
ISD::FSUB, ISD::FMUL, ISD::FMA,
ISD::FDIV, ISD::FSQRT, ISD::FABS,
ISD::FNEG, ISD::STRICT_FMA, ISD::STRICT_FADD,
@@ -711,7 +711,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
- ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
+ ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_FMINIMUMNUM, ISD::VP_FMAXIMUMNUM, ISD::VP_LRINT,
ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
ISD::VP_REDUCE_FMAXIMUM};
@@ -942,7 +942,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
// TODO: support more ops.
static const unsigned ZvfhminPromoteOps[] = {
- ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, ISD::FSUB,
+ ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FADD, ISD::FSUB,
ISD::FMUL, ISD::FMA, ISD::FDIV, ISD::FSQRT,
ISD::FABS, ISD::FNEG, ISD::FCOPYSIGN, ISD::FCEIL,
ISD::FFLOOR, ISD::FROUND, ISD::FROUNDEVEN, ISD::FRINT,
@@ -956,7 +956,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
- ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
+ ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FMINIMUMNUM, ISD::VP_FMAXIMUMNUM, ISD::VP_FCEIL,
ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
@@ -978,6 +978,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setCondCodeAction(VFPCCToExpand, VT, Expand);
setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal);
+ setOperationAction({ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, VT, Legal);
setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
@@ -1356,7 +1357,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
- ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
+ ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
ISD::IS_FPCLASS, ISD::FMAXIMUM, ISD::FMINIMUM},
VT, Custom);
@@ -1447,7 +1448,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::SRA);
if (Subtarget.hasStdExtFOrZfinx())
- setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM});
+ setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM});
if (Subtarget.hasStdExtZbb())
setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN});
@@ -5938,9 +5939,11 @@ static unsigned getRISCVVLOp(SDValue Op) {
case ISD::VP_FP_TO_UINT:
return RISCVISD::VFCVT_RTZ_XU_F_VL;
case ISD::FMINNUM:
+ case ISD::FMINIMUMNUM:
case ISD::VP_FMINNUM:
return RISCVISD::VFMIN_VL;
case ISD::FMAXNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::VP_FMAXNUM:
return RISCVISD::VFMAX_VL;
case ISD::LRINT:
@@ -6881,6 +6884,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::FMA:
case ISD::FMINNUM:
case ISD::FMAXNUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
if (Op.getValueType() == MVT::nxv32f16 &&
(Subtarget.hasVInstructionsF16Minimal() &&
!Subtarget.hasVInstructionsF16()))
@@ -7004,6 +7009,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::VP_FMA:
case ISD::VP_FMINNUM:
case ISD::VP_FMAXNUM:
+ case ISD::VP_FMINIMUMNUM:
+ case ISD::VP_FMAXIMUMNUM:
case ISD::VP_FCOPYSIGN:
if (Op.getValueType() == MVT::nxv32f16 &&
(Subtarget.hasVInstructionsF16Minimal() &&
@@ -12874,8 +12881,10 @@ static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
case ISD::FADD:
return RISCVISD::VECREDUCE_FADD_VL;
case ISD::FMAXNUM:
+ case ISD::FMAXIMUMNUM:
return RISCVISD::VECREDUCE_FMAX_VL;
case ISD::FMINNUM:
+ case ISD::FMINIMUMNUM:
return RISCVISD::VECREDUCE_FMIN_VL;
}
};
@@ -16271,7 +16280,9 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SMAX:
case ISD::SMIN:
case ISD::FMAXNUM:
- case ISD::FMINNUM: {
+ case ISD::FMINNUM:
+ case ISD::FMAXIMUMNUM:
+ case ISD::FMINIMUMNUM: {
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
return V;
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 8efefee383a6a..00e61889d8ad9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -389,6 +389,8 @@ def : Pat<(fneg (any_fma_nsz FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3))
foreach Ext = DExts in {
defm : PatFprFpr_m<fminnum, FMIN_D, Ext>;
defm : PatFprFpr_m<fmaxnum, FMAX_D, Ext>;
+ defm : PatFprFpr_m<fminimumnum, FMIN_D, Ext>;
+ defm : PatFprFpr_m<fmaximumnum, FMAX_D, Ext>;
defm : PatFprFpr_m<riscv_fmin, FMIN_D, Ext>;
defm : PatFprFpr_m<riscv_fmax, FMAX_D, Ext>;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index 7d89608de1223..edc5d2467b2a7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -596,6 +596,8 @@ def : Pat<(fneg (any_fma_nsz FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3)),
foreach Ext = FExts in {
defm : PatFprFpr_m<fminnum, FMIN_S, Ext>;
defm : PatFprFpr_m<fmaxnum, FMAX_S, Ext>;
+ defm : PatFprFpr_m<fminimumnum, FMIN_S, Ext>;
+ defm : PatFprFpr_m<fmaximumnum, FMAX_S, Ext>;
defm : PatFprFpr_m<riscv_fmin, FMIN_S, Ext>;
defm : PatFprFpr_m<riscv_fmax, FMAX_S, Ext>;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 66df24f2a458d..021441e5b6ddf 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1418,6 +1418,8 @@ foreach vti = AllFloatVectors in {
// 13.11. Vector Floating-Point MIN/MAX Instructions
defm : VPatBinaryFPSDNode_VV_VF<fminnum, "PseudoVFMIN", isSEWAware=1>;
defm : VPatBinaryFPSDNode_VV_VF<fmaxnum, "PseudoVFMAX", isSEWAware=1>;
+defm : VPatBinaryFPSDNode_VV_VF<fminimumnum, "PseudoVFMIN", isSEWAware=1>;
+defm : VPatBinaryFPSDNode_VV_VF<fmaximumnum, "PseudoVFMAX", isSEWAware=1>;
// 13.13. Vector Floating-Point Compare Instructions
defm : VPatFPSetCCSDNode_VV_VF_FV<SETEQ, "PseudoVMFEQ", "PseudoVMFEQ">;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 215cbc308e43d..3360a5dd1ba45 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -929,6 +929,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FEXP10, MVT::f80, Expand);
setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
+ setOperationAction(ISD::FMINIMUMNUM, MVT::f80, Expand);
+ setOperationAction(ISD::FMAXIMUMNUM, MVT::f80, Expand);
// Some FP actions are always expanded for vector types.
for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
@@ -2500,6 +2502,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::STRICT_FMA,
ISD::FMINNUM,
ISD::FMAXNUM,
+ ISD::FMINIMUMNUM,
+ ISD::FMAXIMUMNUM,
ISD::SUB,
ISD::LOAD,
ISD::LRINT,
@@ -44586,6 +44590,8 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
case ISD::FMAXNUM_IEEE:
case ISD::FMAXIMUM:
case ISD::FMINIMUM:
+ case ISD::FMAXIMUMNUM:
+ case ISD::FMINIMUMNUM:
case X86ISD::FMAX:
case X86ISD::FMIN:
case ISD::FABS: // Begin 1 operand
@@ -57082,7 +57088,9 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::FMIN:
case X86ISD::FMAX: return combineFMinFMax(N, DAG);
case ISD::FMINNUM:
- case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget);
+ case ISD::FMAXNUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget);
case X86ISD::CVTSI2P:
case X86ISD::CVTUI2P: return combineX86INT_TO_FP(N, DAG, DCI);
case X86ISD::CVTP2SI:
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index ac66144aeaaec..ab2816e480b8b 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -4209,6 +4209,10 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// FMINNUM has same costs so don't duplicate.
ISD = ISD::FMAXNUM;
break;
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum:
+ ISD = ISD::FMAXIMUMNUM;
+ break;
case Intrinsic::sadd_sat:
ISD = ISD::SADDSAT;
break;
@@ -4281,7 +4285,7 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
// If there are no NANs to deal with, then these are reduced to a
// single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
// assume is used in the non-fast case.
- if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
+ if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM || ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
if (FMF.noNaNs())
return LegalizationCost * 1;
}
@@ -5419,7 +5423,10 @@ X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
} else {
assert(ValTy->isFPOrFPVectorTy() &&
"Expected float point or integer vector type.");
- ISD = (IID == Intrinsic::minnum || IID == Intrinsic::maxnum)
+ if (IID == Intrinsic::minimumnum || IID == Intrinsic::maximumnum)
+ ISD = ISD::FMINIMUMNUM;
+ else
+ ISD = (IID == Intrinsic::minnum || IID == Intrinsic::maxnum)
? ISD::FMINNUM
: ISD::FMINIMUM;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 08ecbe304429e..e0d62ef2f514e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17584,6 +17584,8 @@ class HorizontalReduction {
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
+ case RecurKind::FMaximumnum:
+ case RecurKind::FMinimumnum:
// res = vv
return VectorizedValue;
case RecurKind::Mul:
@@ -17646,6 +17648,8 @@ class HorizontalReduction {
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
+ case RecurKind::FMaximumnum:
+ case RecurKind::FMinimumnum:
// No need for multiple min/max(s) of the same value.
LLVM_DEBUG(dbgs() << "SLP: Max/min of same " << VectorizedValue
<< ". (HorRdx)\n");
More information about the llvm-commits
mailing list