[Mlir-commits] [mlir] [mlir][arith] Add minimumnumf and maximumnumf operations (PR #221658)
lorenzo chelini
llvmlistbot at llvm.org
Fri Sep 18 05:50:21 PDT 2026
https://github.com/chelini updated https://github.com/llvm/llvm-project/pull/221658
>From 20574d33ede25b51e9d1041fe2bb742cd3798cca Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Sat, 5 Sep 2026 09:55:21 +0200
Subject: [PATCH] [mlir][arith] Add minimumnumf and maximumnumf operations
Add `arith.minimumnumf` and `arith.maximumnumf`, following `minimumNumber` and
`maximumNumber` from IEEE 754-2019 and the `llvm.minimumnum`/`llvm.maximumnum`
intrinsics. They differ from the existing `arith.minnumf`/`arith.maxnumf` only
in their treatment of signaling NaNs: the new ops deterministically return the
non-NaN operand, whereas minnumf/maxnumf may return either NaN or the other
operand.
---
.../include/mlir/Dialect/Arith/IR/ArithOps.td | 52 ++++++
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 4 +
.../Dialect/Vector/IR/VectorAttributes.td | 5 +-
.../Conversion/ArithToLLVM/ArithToLLVM.cpp | 10 ++
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 80 +++++++++
mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 34 ++++
.../Linalg/Transforms/Vectorization.cpp | 4 +
mlir/lib/Dialect/Vector/IR/VectorOps.cpp | 12 ++
.../Vector/Transforms/LowerVectorContract.cpp | 4 +-
.../Vector/Transforms/LowerVectorScan.cpp | 2 +
mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp | 2 +
.../Conversion/ArithToLLVM/arith-to-llvm.mlir | 57 +++++-
.../vector-reduction-to-llvm.mlir | 130 ++++++++++++++
.../vector-to-llvm-interface.mlir | 24 +++
mlir/test/Dialect/Arith/ops.mlir | 7 +-
mlir/test/Dialect/Vector/ops.mlir | 4 +
.../CPU/reductions-masked-minmaxnum-f32.mlir | 166 ++++++++++++++++++
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 28 +++
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 22 +++
19 files changed, 640 insertions(+), 7 deletions(-)
create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmaxnum-f32.mlir
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 39d6838ae1ab4..219fb12ab4ed1 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -1190,6 +1190,32 @@ def Arith_MaxNumFOp : Arith_FloatBinaryOp<"maxnumf", [Commutative]> {
let hasFolder = 1;
}
+//===----------------------------------------------------------------------===//
+// MaximumNumFOp
+//===----------------------------------------------------------------------===//
+
+def Arith_MaximumNumFOp : Arith_FloatBinaryOp<"maximumnumf", [Commutative]> {
+ let summary = "floating-point maximumNumber operation";
+ let description = [{
+ Returns the maximum of the two arguments, not propagating NaNs and treating
+ -0.0 as less than +0.0.
+
+ If both operands are NaNs, returns a NaN. If one operand is NaN and the
+ other operand is a number, returns the number. Otherwise, returns the
+ greater of the two arguments. -0.0 is considered to be less than +0.0 for
+ this operation.
+
+ If the `nsz` flag is specified, `arith.maximumnumf` with one +0.0 and one
+ -0.0 operand may non-deterministically return either operand. Unlike the
+ usual `nsz` semantics, if both zero operands have the same sign, the result
+ must retain that sign. For example, `maximumnumf(+0.0, +0.0)` with `nsz`
+ must return +0.0, while `maximumnumf(-0.0, +0.0)` may return either zero.
+
+ This operation follows the semantics of `maximumNumber` in IEEE 754-2019.
+ }];
+ let hasFolder = 1;
+}
+
//===----------------------------------------------------------------------===//
// MaxSIOp
@@ -1252,6 +1278,32 @@ def Arith_MinNumFOp : Arith_FloatBinaryOp<"minnumf", [Commutative]> {
let hasFolder = 1;
}
+//===----------------------------------------------------------------------===//
+// MinimumNumFOp
+//===----------------------------------------------------------------------===//
+
+def Arith_MinimumNumFOp : Arith_FloatBinaryOp<"minimumnumf", [Commutative]> {
+ let summary = "floating-point minimumNumber operation";
+ let description = [{
+ Returns the minimum of the two arguments, not propagating NaNs and treating
+ -0.0 as less than +0.0.
+
+ If both operands are NaNs, returns a NaN. If one operand is NaN and the
+ other operand is a number, returns the number. Otherwise, returns the
+ lesser of the two arguments. -0.0 is considered to be less than +0.0 for
+ this operation.
+
+ If the `nsz` flag is specified, `arith.minimumnumf` with one +0.0 and one
+ -0.0 operand may non-deterministically return either operand. Unlike the
+ usual `nsz` semantics, if both zero operands have the same sign, the result
+ must retain that sign. For example, `minimumnumf(-0.0, -0.0)` with `nsz`
+ must return -0.0, while `minimumnumf(-0.0, +0.0)` may return either zero.
+
+ This operation follows the semantics of `minimumNumber` in IEEE 754-2019.
+ }];
+ let hasFolder = 1;
+}
+
//===----------------------------------------------------------------------===//
// MinSIOp
//===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 785dd40a174ff..1704005a8e5b1 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -201,6 +201,8 @@ def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrOpF<"maxnum">;
def LLVM_MinNumOp : LLVM_BinarySameArgsIntrOpF<"minnum">;
def LLVM_MaximumOp : LLVM_BinarySameArgsIntrOpF<"maximum">;
def LLVM_MinimumOp : LLVM_BinarySameArgsIntrOpF<"minimum">;
+def LLVM_MaximumNumOp : LLVM_BinarySameArgsIntrOpF<"maximumnum">;
+def LLVM_MinimumNumOp : LLVM_BinarySameArgsIntrOpF<"minimumnum">;
def LLVM_SMaxOp : LLVM_BinarySameArgsIntrOpI<"smax">;
def LLVM_SMinOp : LLVM_BinarySameArgsIntrOpI<"smin">;
def LLVM_UMaxOp : LLVM_BinarySameArgsIntrOpI<"umax">;
@@ -1282,6 +1284,8 @@ def LLVM_vector_reduce_fmax : LLVM_VecReductionF<"fmax">;
def LLVM_vector_reduce_fmin : LLVM_VecReductionF<"fmin">;
def LLVM_vector_reduce_fmaximum : LLVM_VecReductionF<"fmaximum">;
def LLVM_vector_reduce_fminimum : LLVM_VecReductionF<"fminimum">;
+def LLVM_vector_reduce_fmaximumnum : LLVM_VecReductionF<"fmaximumnum">;
+def LLVM_vector_reduce_fminimumnum : LLVM_VecReductionF<"fminimumnum">;
def LLVM_vector_reduce_fadd : LLVM_VecReductionAccF<"fadd">;
def LLVM_vector_reduce_fmul : LLVM_VecReductionAccF<"fmul">;
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td b/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td
index 58e18a15e9409..27a64ee2df0c6 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorAttributes.td
@@ -30,6 +30,8 @@ def COMBINING_KIND_OR : I32EnumAttrCase<"OR", 9, "or">;
def COMBINING_KIND_XOR : I32EnumAttrCase<"XOR", 10, "xor">;
def COMBINING_KIND_MINIMUMF : I32EnumAttrCase<"MINIMUMF", 11, "minimumf">;
def COMBINING_KIND_MAXIMUMF : I32EnumAttrCase<"MAXIMUMF", 12, "maximumf">;
+def COMBINING_KIND_MINIMUMNUMF : I32EnumAttrCase<"MINIMUMNUMF", 13, "minimumnumf">;
+def COMBINING_KIND_MAXIMUMNUMF : I32EnumAttrCase<"MAXIMUMNUMF", 14, "maximumnumf">;
def CombiningKind : I32EnumAttr<
"CombiningKind",
@@ -38,7 +40,8 @@ def CombiningKind : I32EnumAttr<
COMBINING_KIND_MINSI, COMBINING_KIND_MINNUMF, COMBINING_KIND_MAXUI,
COMBINING_KIND_MAXSI, COMBINING_KIND_MAXNUMF, COMBINING_KIND_AND,
COMBINING_KIND_OR, COMBINING_KIND_XOR,
- COMBINING_KIND_MAXIMUMF, COMBINING_KIND_MINIMUMF]> {
+ COMBINING_KIND_MAXIMUMF, COMBINING_KIND_MINIMUMF,
+ COMBINING_KIND_MAXIMUMNUMF, COMBINING_KIND_MINIMUMNUMF]> {
let cppNamespace = "::mlir::vector";
let genSpecializedAttr = 0;
}
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index 982b1e5e9a8d1..198b1490f0046 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -139,6 +139,10 @@ using MaxNumFOpLowering =
VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
arith::AttrConvertFastMathToLLVM,
/*FailOnUnsupportedFP=*/true>;
+using MaximumNumFOpLowering =
+ VectorConvertToLLVMPattern<arith::MaximumNumFOp, LLVM::MaximumNumOp,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
using MaxSIOpLowering =
VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>;
using MaxUIOpLowering =
@@ -151,6 +155,10 @@ using MinNumFOpLowering =
VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
arith::AttrConvertFastMathToLLVM,
/*FailOnUnsupportedFP=*/true>;
+using MinimumNumFOpLowering =
+ VectorConvertToLLVMPattern<arith::MinimumNumFOp, LLVM::MinimumNumOp,
+ arith::AttrConvertFastMathToLLVM,
+ /*FailOnUnsupportedFP=*/true>;
using MinSIOpLowering =
VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>;
using MinUIOpLowering =
@@ -884,10 +892,12 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
IndexCastOpUILowering,
MaximumFOpLowering,
MaxNumFOpLowering,
+ MaximumNumFOpLowering,
MaxSIOpLowering,
MaxUIOpLowering,
MinimumFOpLowering,
MinNumFOpLowering,
+ MinimumNumFOpLowering,
MinSIOpLowering,
MinUIOpLowering,
MulFOpLowering,
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index dd62377c2f6d3..31f5634ef970f 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -726,6 +726,14 @@ template <>
struct VectorToScalarMapper<LLVM::vector_reduce_fmin> {
using Type = LLVM::MinNumOp;
};
+template <>
+struct VectorToScalarMapper<LLVM::vector_reduce_fmaximumnum> {
+ using Type = LLVM::MaximumNumOp;
+};
+template <>
+struct VectorToScalarMapper<LLVM::vector_reduce_fminimumnum> {
+ using Type = LLVM::MinimumNumOp;
+};
} // namespace
template <class LLVMRedIntrinOp>
@@ -743,6 +751,60 @@ static Value createFPReductionComparisonOpLowering(
return result;
}
+/// Mask neutral classes for overloading.
+class MaskNeutralFMaximumNum {};
+class MaskNeutralFMinimumNum {};
+
+/// Get the mask neutral value for a `fmaximumnum` reduction. `maximumnum`
+/// ignores NaN operands, making a quiet NaN the identity element. When `nnan`
+/// promises no NaN reaches the reduction, fall back to the smallest
+/// representable value, which `ninf` further constrains to be finite.
+static llvm::APFloat getMaskNeutralValue(MaskNeutralFMaximumNum,
+ const llvm::fltSemantics &semantics,
+ bool noNaNs, bool noInfs) {
+ if (!noNaNs)
+ return llvm::APFloat::getQNaN(semantics, /*Negative=*/false);
+ return noInfs ? llvm::APFloat::getLargest(semantics, /*Negative=*/true)
+ : llvm::APFloat::getInf(semantics, /*Negative=*/true);
+}
+
+/// Get the mask neutral value for a `fminimumnum` reduction. See the
+/// `fmaximumnum` overload above for the rationale.
+static llvm::APFloat getMaskNeutralValue(MaskNeutralFMinimumNum,
+ const llvm::fltSemantics &semantics,
+ bool noNaNs, bool noInfs) {
+ if (!noNaNs)
+ return llvm::APFloat::getQNaN(semantics, /*Negative=*/false);
+ return noInfs ? llvm::APFloat::getLargest(semantics, /*Negative=*/false)
+ : llvm::APFloat::getInf(semantics, /*Negative=*/false);
+}
+
+/// Lowers masked `fmaximumnum` and `fminimumnum` reductions using the
+/// non-masked intrinsics, since LLVM has no predicated counterpart for them.
+/// Inactive lanes are replaced by a mask neutral value before reducing.
+/// TODO: Switch to `lowerPredicatedReductionWithStartValue` once
+/// `llvm.vp.reduce.fmaximumnum`/`fminimumnum` are added to LLVM IR.
+template <class LLVMRedIntrinOp, class MaskNeutral>
+static Value
+lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType,
+ Value vectorOperand, Value accumulator,
+ Value mask, LLVM::FastmathFlagsAttr fmf) {
+ const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
+ auto value = getMaskNeutralValue(
+ MaskNeutral{}, floatSemantics,
+ LLVM::bitEnumContainsAny(fmf.getValue(), LLVM::FastmathFlags::nnan),
+ LLVM::bitEnumContainsAny(fmf.getValue(), LLVM::FastmathFlags::ninf));
+ Type vectorType = vectorOperand.getType();
+ auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
+ const Value vectorMaskNeutral =
+ LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
+ const Value selectedVectorByMask = LLVM::SelectOp::create(
+ rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
+ return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
+ rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
+}
+
template <class LLVMRedIntrinOp, class ReductionNeutral>
static Value
lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
@@ -901,6 +963,14 @@ class VectorReductionOpConversion
} else if (kind == vector::CombiningKind::MAXNUMF) {
result = createFPReductionComparisonOpLowering<LLVM::vector_reduce_fmax>(
rewriter, loc, llvmType, operand, acc, fmf);
+ } else if (kind == vector::CombiningKind::MAXIMUMNUMF) {
+ result = createFPReductionComparisonOpLowering<
+ LLVM::vector_reduce_fmaximumnum>(rewriter, loc, llvmType, operand,
+ acc, fmf);
+ } else if (kind == vector::CombiningKind::MINIMUMNUMF) {
+ result = createFPReductionComparisonOpLowering<
+ LLVM::vector_reduce_fminimumnum>(rewriter, loc, llvmType, operand,
+ acc, fmf);
} else {
return failure();
}
@@ -1051,6 +1121,16 @@ class MaskedReductionOpConversion
LLVM::VPReduceFMinimumOp, ReductionNeutralFPPosInf>(
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
+ case CombiningKind::MAXIMUMNUMF:
+ result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximumnum,
+ MaskNeutralFMaximumNum>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ break;
+ case CombiningKind::MINIMUMNUMF:
+ result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimumnum,
+ MaskNeutralFMinimumNum>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ break;
}
// Replace `vector.mask` operation altogether.
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 95ec15e515eeb..01d0c18670cb8 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -1438,6 +1438,22 @@ OpFoldResult arith::MaxNumFOp::fold(FoldAdaptor adaptor) {
return constFoldBinaryOp<FloatAttr>(adaptor.getOperands(), llvm::maxnum);
}
+//===----------------------------------------------------------------------===//
+// MaximumNumFOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult arith::MaximumNumFOp::fold(FoldAdaptor adaptor) {
+ // maximumnumf(x,x) -> x
+ if (getLhs() == getRhs())
+ return getRhs();
+
+ // maximumnumf(x, NaN) -> x
+ if (matchPattern(adaptor.getRhs(), m_NaNFloat()))
+ return getLhs();
+
+ return constFoldBinaryOp<FloatAttr>(adaptor.getOperands(), llvm::maximumnum);
+}
+
//===----------------------------------------------------------------------===//
// MaxSIOp
//===----------------------------------------------------------------------===//
@@ -1516,6 +1532,22 @@ OpFoldResult arith::MinNumFOp::fold(FoldAdaptor adaptor) {
return constFoldBinaryOp<FloatAttr>(adaptor.getOperands(), llvm::minnum);
}
+//===----------------------------------------------------------------------===//
+// MinimumNumFOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult arith::MinimumNumFOp::fold(FoldAdaptor adaptor) {
+ // minimumnumf(x,x) -> x
+ if (getLhs() == getRhs())
+ return getRhs();
+
+ // minimumnumf(x, NaN) -> x
+ if (matchPattern(adaptor.getRhs(), m_NaNFloat()))
+ return getLhs();
+
+ return constFoldBinaryOp<FloatAttr>(adaptor.getOperands(), llvm::minimumnum);
+}
+
//===----------------------------------------------------------------------===//
// MinSIOp
//===----------------------------------------------------------------------===//
@@ -2069,8 +2101,10 @@ void arith::TruncFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
MLIRContext *context) {
patterns.add<NarrowExtremum<TruncFOp, ExtFOp, MaximumFOp>,
NarrowExtremum<TruncFOp, ExtFOp, MaxNumFOp>,
+ NarrowExtremum<TruncFOp, ExtFOp, MaximumNumFOp>,
NarrowExtremum<TruncFOp, ExtFOp, MinimumFOp>,
NarrowExtremum<TruncFOp, ExtFOp, MinNumFOp>,
+ NarrowExtremum<TruncFOp, ExtFOp, MinimumNumFOp>,
TruncFSIToFPToSIToFP, TruncFUIToFPToUIToFP>(context);
}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index bb620641951dc..46d2f2f367c17 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -634,10 +634,12 @@ mlir::linalg::getCombinerOpKind(Operation *combinerOp) {
.Case([&](arith::MaxUIOp op) { return CombiningKind::MAXUI; })
.Case([&](arith::MaximumFOp op) { return CombiningKind::MAXIMUMF; })
.Case([&](arith::MaxNumFOp op) { return CombiningKind::MAXNUMF; })
+ .Case([&](arith::MaximumNumFOp op) { return CombiningKind::MAXIMUMNUMF; })
.Case([&](arith::MinSIOp op) { return CombiningKind::MINSI; })
.Case([&](arith::MinUIOp op) { return CombiningKind::MINUI; })
.Case([&](arith::MinimumFOp op) { return CombiningKind::MINIMUMF; })
.Case([&](arith::MinNumFOp op) { return CombiningKind::MINNUMF; })
+ .Case([&](arith::MinimumNumFOp op) { return CombiningKind::MINIMUMNUMF; })
.Case<arith::MulIOp, arith::MulFOp>(
[&](auto op) { return CombiningKind::MUL; })
.Case([&](arith::OrIOp op) { return CombiningKind::OR; })
@@ -2189,10 +2191,12 @@ static bool isSupportedPoolKind(vector::CombiningKind kind) {
case vector::CombiningKind::ADD:
case vector::CombiningKind::MAXNUMF:
case vector::CombiningKind::MAXIMUMF:
+ case vector::CombiningKind::MAXIMUMNUMF:
case vector::CombiningKind::MAXSI:
case vector::CombiningKind::MAXUI:
case vector::CombiningKind::MINNUMF:
case vector::CombiningKind::MINIMUMF:
+ case vector::CombiningKind::MINIMUMNUMF:
case vector::CombiningKind::MINSI:
case vector::CombiningKind::MINUI:
return true;
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index b77796a42232f..5ee41f070e320 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -152,6 +152,8 @@ static bool isSupportedCombiningKind(CombiningKind combiningKind,
case CombiningKind::MAXNUMF:
case CombiningKind::MINIMUMF:
case CombiningKind::MAXIMUMF:
+ case CombiningKind::MINIMUMNUMF:
+ case CombiningKind::MAXIMUMNUMF:
return llvm::isa<FloatType>(elementType);
}
return false;
@@ -8363,6 +8365,11 @@ Value mlir::vector::makeArithReduction(OpBuilder &b, Location loc,
"expected float values");
result = b.createOrFold<arith::MaxNumFOp>(loc, v1, acc, fastmath);
break;
+ case CombiningKind::MAXIMUMNUMF:
+ assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
+ "expected float values");
+ result = b.createOrFold<arith::MaximumNumFOp>(loc, v1, acc, fastmath);
+ break;
case CombiningKind::MAXIMUMF:
assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
"expected float values");
@@ -8373,6 +8380,11 @@ Value mlir::vector::makeArithReduction(OpBuilder &b, Location loc,
"expected float values");
result = b.createOrFold<arith::MinNumFOp>(loc, v1, acc, fastmath);
break;
+ case CombiningKind::MINIMUMNUMF:
+ assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
+ "expected float values");
+ result = b.createOrFold<arith::MinimumNumFOp>(loc, v1, acc, fastmath);
+ break;
case CombiningKind::MINIMUMF:
assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
"expected float values");
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index 2a152ef0634d7..b3cd4fb1e8f9b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -148,7 +148,9 @@ createContractArithOp(Location loc, Value x, Value y, Value acc,
if (isInt) {
if (kind == CombiningKind::MINNUMF || kind == CombiningKind::MAXNUMF ||
- kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF)
+ kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF ||
+ kind == CombiningKind::MINIMUMNUMF ||
+ kind == CombiningKind::MAXIMUMNUMF)
// Only valid for floating point types.
return std::nullopt;
mul = arith::MulIOp::create(rewriter, loc, x, y);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
index f3e239bedb7f7..c3ea8de30357e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorScan.cpp
@@ -38,6 +38,8 @@ static bool isValidKind(bool isInt, vector::CombiningKind kind) {
case CombiningKind::MINIMUMF:
case CombiningKind::MAXNUMF:
case CombiningKind::MAXIMUMF:
+ case CombiningKind::MAXIMUMNUMF:
+ case CombiningKind::MINIMUMNUMF:
type = KindType::FLOAT;
break;
case CombiningKind::MINUI:
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index bd2fe1e7f1a6b..1a3d412a752a3 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -765,6 +765,7 @@ Value xegpu::createReductionNeutralValue(OpBuilder &builder, Location loc,
case vector::CombiningKind::MINNUMF:
case vector::CombiningKind::MINIMUMF:
+ case vector::CombiningKind::MINIMUMNUMF:
if (auto floatTy = dyn_cast<FloatType>(elemTy))
return makeConst(builder.getFloatAttr(
elemTy, APFloat::getInf(floatTy.getFloatSemantics())));
@@ -772,6 +773,7 @@ Value xegpu::createReductionNeutralValue(OpBuilder &builder, Location loc,
case vector::CombiningKind::MAXNUMF:
case vector::CombiningKind::MAXIMUMF:
+ case vector::CombiningKind::MAXIMUMNUMF:
if (auto floatTy = dyn_cast<FloatType>(elemTy))
return makeConst(builder.getFloatAttr(
elemTy, APFloat::getInf(floatTy.getFloatSemantics(), true)));
diff --git a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
index f2f3d60619fa2..ebe712e4f0b15 100644
--- a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
+++ b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
@@ -838,11 +838,56 @@ func.func @minmaxf(%arg0 : f32, %arg1 : f32) -> f32 {
%2 = arith.minnumf %arg0, %arg1 : f32
// CHECK: = llvm.intr.maxnum(%arg0, %arg1) : (f32, f32) -> f32
%3 = arith.maxnumf %arg0, %arg1 : f32
+ // CHECK: = llvm.intr.minimumnum(%arg0, %arg1) : (f32, f32) -> f32
+ %4 = arith.minimumnumf %arg0, %arg1 : f32
+ // CHECK: = llvm.intr.maximumnum(%arg0, %arg1) : (f32, f32) -> f32
+ %5 = arith.maximumnumf %arg0, %arg1 : f32
return %0 : f32
}
// -----
+// CHECK-LABEL: @minmaxnumf_float_widths
+func.func @minmaxnumf_float_widths(%arg0 : f16, %arg1 : f16, %arg2 : bf16,
+ %arg3 : bf16, %arg4 : f64, %arg5 : f64) {
+ // CHECK: = llvm.intr.minimumnum(%arg0, %arg1) : (f16, f16) -> f16
+ %0 = arith.minimumnumf %arg0, %arg1 : f16
+ // CHECK: = llvm.intr.maximumnum(%arg0, %arg1) : (f16, f16) -> f16
+ %1 = arith.maximumnumf %arg0, %arg1 : f16
+ // CHECK: = llvm.intr.minimumnum(%arg2, %arg3) : (bf16, bf16) -> bf16
+ %2 = arith.minimumnumf %arg2, %arg3 : bf16
+ // CHECK: = llvm.intr.maximumnum(%arg2, %arg3) : (bf16, bf16) -> bf16
+ %3 = arith.maximumnumf %arg2, %arg3 : bf16
+ // CHECK: = llvm.intr.minimumnum(%arg4, %arg5) : (f64, f64) -> f64
+ %4 = arith.minimumnumf %arg4, %arg5 : f64
+ // CHECK: = llvm.intr.maximumnum(%arg4, %arg5) : (f64, f64) -> f64
+ %5 = arith.maximumnumf %arg4, %arg5 : f64
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @minmaxnumf_vector
+func.func @minmaxnumf_vector(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) {
+ // CHECK: = llvm.intr.minimumnum(%arg0, %arg1) : (vector<4xf32>, vector<4xf32>) -> vector<4xf32>
+ %0 = arith.minimumnumf %arg0, %arg1 : vector<4xf32>
+ // CHECK: = llvm.intr.maximumnum(%arg0, %arg1) : (vector<4xf32>, vector<4xf32>) -> vector<4xf32>
+ %1 = arith.maximumnumf %arg0, %arg1 : vector<4xf32>
+ return
+}
+
+// -----
+
+// CHECK-LABEL: @minmaxnumf_2dvector
+// CHECK-COUNT-2: llvm.intr.minimumnum({{.*}}) : (vector<4xf32>, vector<4xf32>) -> vector<4xf32>
+// CHECK-COUNT-2: llvm.intr.maximumnum({{.*}}) : (vector<4xf32>, vector<4xf32>) -> vector<4xf32>
+func.func @minmaxnumf_2dvector(%arg0 : vector<2x4xf32>,
+ %arg1 : vector<2x4xf32>) {
+ %0 = arith.minimumnumf %arg0, %arg1 : vector<2x4xf32>
+ %1 = arith.maximumnumf %arg0, %arg1 : vector<2x4xf32>
+ return
+}
+
// CHECK-LABEL: @fastmath
func.func @fastmath(%arg0: f32, %arg1: f32, %arg2: i32) {
// CHECK: llvm.fadd %arg0, %arg1 fastmath<fast> : f32
@@ -870,14 +915,18 @@ func.func @ops_supporting_fastmath(%arg0: f32, %arg1: f32, %arg2: i32) {
%2 = arith.maximumf %arg0, %arg1 fastmath<fast> : f32
// CHECK: llvm.intr.minimum(%arg0, %arg1) fastmath<fast> : (f32, f32) -> f32
%3 = arith.minimumf %arg0, %arg1 fastmath<fast> : f32
+// CHECK: llvm.intr.maximumnum(%arg0, %arg1) fastmath<fast> : (f32, f32) -> f32
+ %4 = arith.maximumnumf %arg0, %arg1 fastmath<fast> : f32
+// CHECK: llvm.intr.minimumnum(%arg0, %arg1) fastmath<fast> : (f32, f32) -> f32
+ %5 = arith.minimumnumf %arg0, %arg1 fastmath<fast> : f32
// CHECK: llvm.fmul %arg0, %arg1 fastmath<fast> : f32
- %4 = arith.mulf %arg0, %arg1 fastmath<fast> : f32
+ %6 = arith.mulf %arg0, %arg1 fastmath<fast> : f32
// CHECK: llvm.fneg %arg0 fastmath<fast> : f32
- %5 = arith.negf %arg0 fastmath<fast> : f32
+ %7 = arith.negf %arg0 fastmath<fast> : f32
// CHECK: llvm.frem %arg0, %arg1 fastmath<fast> : f32
- %6 = arith.remf %arg0, %arg1 fastmath<fast> : f32
+ %8 = arith.remf %arg0, %arg1 fastmath<fast> : f32
// CHECK: llvm.fsub %arg0, %arg1 fastmath<fast> : f32
- %7 = arith.subf %arg0, %arg1 fastmath<fast> : f32
+ %9 = arith.subf %arg0, %arg1 fastmath<fast> : f32
return
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
index 6ac19a0e5f4fb..5117e5245e2a5 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -246,6 +246,136 @@ func.func @masked_reduce_minimumf_f32_fast(%arg0: vector<16xf32>, %mask : vector
// -----
+// `maximumnum`/`minimumnum` have no predicated LLVM intrinsic, so inactive
+// lanes are filled with a mask neutral value before an unmasked reduction.
+// The neutral is a quiet NaN, which these operations ignore.
+
+func.func @masked_reduce_maximumnumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumnumf>, %arg0 : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumnumf_f32(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0x7FC00000> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximumnum(%[[MASKED]]) : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+func.func @masked_reduce_minimumnumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumnumf>, %arg0 : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumnumf_f32(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0x7FC00000> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimumnum(%[[MASKED]]) : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+// Under `nnan` a NaN neutral would be poison, so the neutral becomes an
+// infinity instead.
+
+func.func @masked_reduce_maximumnumf_f32_nnan(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumnumf>, %arg0 fastmath<nnan> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumnumf_f32_nnan(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0xFF800000> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximumnum(%[[MASKED]]) fastmath<nnan> : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+func.func @masked_reduce_minimumnumf_f32_nnan(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumnumf>, %arg0 fastmath<nnan> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumnumf_f32_nnan(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0x7F800000> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimumnum(%[[MASKED]]) fastmath<nnan> : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+// `fast` is a group that includes `nnan` and `ninf`, so it selects the finite
+// neutral.
+
+func.func @masked_reduce_maximumnumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumnumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumnumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<-3.40282347E+38> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximumnum(%[[MASKED]]) fastmath<fast> : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+func.func @masked_reduce_minimumnumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumnumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumnumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<3.40282347E+38> : vector<16xf32>) : vector<16xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<16xi1>, vector<16xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimumnum(%[[MASKED]]) fastmath<fast> : (vector<16xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+func.func @masked_reduce_maximumnumf_f32_scalable(%arg0: vector<[16]xf32>, %mask : vector<[16]xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumnumf>, %arg0 : vector<[16]xf32> into f32 } : vector<[16]xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumnumf_f32_scalable(
+// CHECK-SAME: %[[INPUT:.*]]: vector<[16]xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<[16]xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0x7FC00000> : vector<[16]xf32>) : vector<[16]xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<[16]xi1>, vector<[16]xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximumnum(%[[MASKED]]) : (vector<[16]xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
+func.func @masked_reduce_minimumnumf_f32_scalable(%arg0: vector<[16]xf32>, %mask : vector<[16]xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumnumf>, %arg0 : vector<[16]xf32> into f32 } : vector<[16]xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumnumf_f32_scalable(
+// CHECK-SAME: %[[INPUT:.*]]: vector<[16]xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<[16]xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(dense<0x7FC00000> : vector<[16]xf32>) : vector<[16]xf32>
+// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[NEUTRAL]] : vector<[16]xi1>, vector<[16]xf32>
+// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimumnum(%[[MASKED]]) : (vector<[16]xf32>) -> f32
+// CHECK: return %[[RESULT]]
+
+// -----
+
func.func @masked_reduce_add_i8(%arg0: vector<32xi8>, %mask : vector<32xi1>) -> i8 {
%0 = vector.mask %mask { vector.reduction <add>, %arg0 : vector<32xi8> into i8 } : vector<32xi1> -> i8
return %0 : i8
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 95a002545d2d4..4d5ba3c22f91e 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2053,6 +2053,30 @@ func.func @scalable_insert(%sub: vector<4xf32>, %dsv: vector<[4]xf32>) -> vector
// -----
+func.func @reduce_fminimumnum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+ %0 = vector.reduction <minimumnumf>, %arg0, %arg1 : vector<16xf32> into f32
+ return %0 : f32
+}
+// CHECK-LABEL: @reduce_fminimumnum_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+// CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fminimumnum(%[[A]]) : (vector<16xf32>) -> f32
+// CHECK: %[[R:.*]] = llvm.intr.minimumnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+// CHECK: return %[[R]] : f32
+
+// -----
+
+func.func @reduce_fmaximumnum_f32(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
+ %0 = vector.reduction <maximumnumf>, %arg0, %arg1 : vector<16xf32> into f32
+ return %0 : f32
+}
+// CHECK-LABEL: @reduce_fmaximumnum_f32(
+// CHECK-SAME: %[[A:.*]]: vector<16xf32>, %[[B:.*]]: f32)
+// CHECK: %[[V:.*]] = llvm.intr.vector.reduce.fmaximumnum(%[[A]]) : (vector<16xf32>) -> f32
+// CHECK: %[[R:.*]] = llvm.intr.maximumnum(%[[V]], %[[B]]) : (f32, f32) -> f32
+// CHECK: return %[[R]] : f32
+
+// -----
+
//===----------------------------------------------------------------------===//
// vector.scalable_extract
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Arith/ops.mlir b/mlir/test/Dialect/Arith/ops.mlir
index 76aef203a64fb..1fcb092422cae 100644
--- a/mlir/test/Dialect/Arith/ops.mlir
+++ b/mlir/test/Dialect/Arith/ops.mlir
@@ -1241,6 +1241,9 @@ func.func @maximum(%v1: vector<4xf32>, %v2: vector<4xf32>,
%maxnum_vector = arith.maxnumf %v1, %v2 : vector<4xf32>
%maxnum_scalable_vector = arith.maxnumf %sv1, %sv2 : vector<[4]xf32>
%maxnum_float = arith.maxnumf %f1, %f2 : f32
+ %maximumnum_vector = arith.maximumnumf %v1, %v2 : vector<4xf32>
+ %maximumnum_scalable_vector = arith.maximumnumf %sv1, %sv2 : vector<[4]xf32>
+ %maximumnum_float = arith.maximumnumf %f1, %f2 : f32
%max_signed = arith.maxsi %i1, %i2 : i32
%max_unsigned = arith.maxui %i1, %i2 : i32
return
@@ -1257,6 +1260,9 @@ func.func @minimum(%v1: vector<4xf32>, %v2: vector<4xf32>,
%minnum_vector = arith.minnumf %v1, %v2 : vector<4xf32>
%minnum_scalable_vector = arith.minnumf %sv1, %sv2 : vector<[4]xf32>
%minnum_float = arith.minnumf %f1, %f2 : f32
+ %minimumnum_vector = arith.minimumnumf %v1, %v2 : vector<4xf32>
+ %minimumnum_scalable_vector = arith.minimumnumf %sv1, %sv2 : vector<[4]xf32>
+ %minimumnum_float = arith.minimumnumf %f1, %f2 : f32
%min_signed = arith.minsi %i1, %i2 : i32
%min_unsigned = arith.minui %i1, %i2 : i32
return
@@ -1334,4 +1340,3 @@ func.func @intflags_func(%arg0: i64, %arg1: i64) {
%4 = arith.trunci %arg0 overflow<nsw, nuw> : i64 to i32
return
}
-
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index 96e844b0384f8..bd123bd0a6bd8 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -624,6 +624,10 @@ func.func @reduce_fp(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
vector.reduction <minimumf>, %arg0 : vector<16xf32> into f32
// CHECK: %[[X1:.*]] = vector.reduction <maximumf>, %{{.*}} : vector<16xf32> into f32
%1 = vector.reduction <maximumf>, %arg0 : vector<16xf32> into f32
+ // CHECK: vector.reduction <minimumnumf>, %{{.*}} : vector<16xf32> into f32
+ vector.reduction <minimumnumf>, %arg0 : vector<16xf32> into f32
+ // CHECK: %[[X2:.*]] = vector.reduction <maximumnumf>, %{{.*}} : vector<16xf32> into f32
+ %2 = vector.reduction <maximumnumf>, %arg0 : vector<16xf32> into f32
// CHECK: return %[[X0]] : f32
return %0 : f32
}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmaxnum-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmaxnum-f32.mlir
new file mode 100644
index 0000000000000..b76da826b4a3e
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmaxnum-f32.mlir
@@ -0,0 +1,166 @@
+// RUN: mlir-opt %s -test-lower-to-llvm | \
+// RUN: mlir-runner -e entry -entry-point-result=void \
+// RUN: -shared-libs=%mlir_c_runner_utils | \
+// RUN: FileCheck %s
+
+func.func @maximumnumf_finite() {
+ // The neutral (qNaN) is ignored by maximumnum. The masked-off lane is the
+ // largest.
+ // maximumnum(<qNaN>, -5, -7, -5) = -5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_finite
+// CHECK-NEXT: -5
+
+func.func @minimumnumf_finite() {
+ // minimumnum(<qNaN>, 5, 7, 5) = 5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumnumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumnumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumnumf_finite
+// CHECK-NEXT: 5
+
+func.func @maximumnumf_nan_active_lane() {
+ // A NaN in an active lane is ignored, unlike maximumf.
+ // maximumnum(<qNaN>, qNaN, -7, -5) = -5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[0x7FC00000, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_nan_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_nan_active_lane
+// CHECK-NEXT: -5
+
+func.func @maximumnumf_no_active_lane() {
+ // With no active lane the neutral wins.
+ // maximumnum(<qNaN>) = qNaN
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_no_active_lane
+// CHECK-NEXT: nan
+
+func.func @minimumnumf_no_active_lane() {
+ // minimumnum(<qNaN>) = qNaN
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumnumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumnumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumnumf_no_active_lane
+// CHECK-NEXT: nan
+
+func.func @maximumnumf_nnan_no_active_lane() {
+ // Under `nnan` the neutral is -inf instead of qNaN.
+ // maximumnum(<-inf>) = -inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v fastmath<nnan> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_nnan_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_nnan_no_active_lane
+// CHECK-NEXT: -inf
+
+func.func @minimumnumf_nnan_no_active_lane() {
+ // minimumnum(<+inf>) = +inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumnumf>, %v fastmath<nnan> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumnumf_nnan_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumnumf_nnan_no_active_lane
+// CHECK-NEXT: inf
+
+func.func @maximumnumf_nnan_ninf_no_active_lane() {
+ // Under `nnan, ninf` the neutral is -FLT_MAX.
+ // maximumnum(<-FLT_MAX>) = -FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v fastmath<nnan,ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_nnan_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_nnan_ninf_no_active_lane
+// CHECK-NEXT: -3.40282e+38
+
+func.func @minimumnumf_nnan_ninf_no_active_lane() {
+ // minimumnum(<+FLT_MAX>) = +FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumnumf>, %v fastmath<nnan,ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumnumf_nnan_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumnumf_nnan_ninf_no_active_lane
+// CHECK-NEXT: 3.40282e+38
+
+func.func @maximumnumf_acc() {
+ // An accumulator takes the place of the neutral.
+ // maximumnum(<-3>, -5, -7, -5) = -3
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %acc = arith.constant -3.0 : f32
+ %0 = vector.mask %mask {
+ vector.reduction <maximumnumf>, %v, %acc : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumnumf_acc\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumnumf_acc
+// CHECK-NEXT: -3
+
+func.func @entry() {
+ call @maximumnumf_finite() : () -> ()
+ call @minimumnumf_finite() : () -> ()
+ call @maximumnumf_nan_active_lane() : () -> ()
+ call @maximumnumf_no_active_lane() : () -> ()
+ call @minimumnumf_no_active_lane() : () -> ()
+ call @maximumnumf_nnan_no_active_lane() : () -> ()
+ call @minimumnumf_nnan_no_active_lane() : () -> ()
+ call @maximumnumf_nnan_ninf_no_active_lane() : () -> ()
+ call @minimumnumf_nnan_ninf_no_active_lane() : () -> ()
+ call @maximumnumf_acc() : () -> ()
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index dbc124b339169..62018064b6afa 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -443,6 +443,24 @@ define void @minnum_test(float %0, float %1, <8 x float> %2, <8 x float> %3) {
ret void
}
+; CHECK-LABEL: llvm.func @minimumnum_test
+define void @minimumnum_test(float %0, float %1, <8 x float> %2, <8 x float> %3) {
+ ; CHECK: llvm.intr.minimumnum(%{{.*}}, %{{.*}}) : (f32, f32) -> f32
+ %5 = call float @llvm.minimumnum.f32(float %0, float %1)
+ ; CHECK: llvm.intr.minimumnum(%{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+ %6 = call <8 x float> @llvm.minimumnum.v8f32(<8 x float> %2, <8 x float> %3)
+ ret void
+}
+
+; CHECK-LABEL: llvm.func @maximumnum_test
+define void @maximumnum_test(float %0, float %1, <8 x float> %2, <8 x float> %3) {
+ ; CHECK: llvm.intr.maximumnum(%{{.*}}, %{{.*}}) : (f32, f32) -> f32
+ %5 = call float @llvm.maximumnum.f32(float %0, float %1)
+ ; CHECK: llvm.intr.maximumnum(%{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+ %6 = call <8 x float> @llvm.maximumnum.v8f32(<8 x float> %2, <8 x float> %3)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @smax_test
define void @smax_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) {
; CHECK: llvm.intr.smax(%{{.*}}, %{{.*}}) : (i32, i32) -> i32
@@ -515,6 +533,10 @@ define void @vector_reductions(float %0, <8 x float> %1, <8 x i32> %2) {
%19 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> %1)
; CHECK: llvm.intr.vector.reduce.fminimum(%{{.*}}) : (vector<8xf32>) -> f32
%20 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> %1)
+ ; CHECK: llvm.intr.vector.reduce.fminimumnum(%{{.*}}) : (vector<8xf32>) -> f32
+ %21 = call float @llvm.vector.reduce.fminimumnum.v8f32(<8 x float> %1)
+ ; CHECK: llvm.intr.vector.reduce.fmaximumnum(%{{.*}}) : (vector<8xf32>) -> f32
+ %22 = call float @llvm.vector.reduce.fmaximumnum.v8f32(<8 x float> %1)
ret void
}
@@ -1844,6 +1866,10 @@ declare float @llvm.maxnum.f32(float, float)
declare <8 x float> @llvm.maxnum.v8f32(<8 x float>, <8 x float>)
declare float @llvm.minnum.f32(float, float)
declare <8 x float> @llvm.minnum.v8f32(<8 x float>, <8 x float>)
+declare float @llvm.minimumnum.f32(float, float)
+declare <8 x float> @llvm.minimumnum.v8f32(<8 x float>, <8 x float>)
+declare float @llvm.maximumnum.f32(float, float)
+declare <8 x float> @llvm.maximumnum.v8f32(<8 x float>, <8 x float>)
declare i32 @llvm.smax.i32(i32, i32)
declare <8 x i32> @llvm.smax.v8i32(<8 x i32>, <8 x i32>)
declare i32 @llvm.smin.i32(i32, i32)
@@ -1858,6 +1884,8 @@ declare float @llvm.vector.reduce.fmax.v8f32(<8 x float>)
declare float @llvm.vector.reduce.fmin.v8f32(<8 x float>)
declare float @llvm.vector.reduce.fmaximum.v8f32(<8 x float>)
declare float @llvm.vector.reduce.fminimum.v8f32(<8 x float>)
+declare float @llvm.vector.reduce.fminimumnum.v8f32(<8 x float>)
+declare float @llvm.vector.reduce.fmaximumnum.v8f32(<8 x float>)
declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.or.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.smax.v8i32(<8 x i32>)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index e5a7ec566268e..c0ed71aac90e4 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -468,6 +468,24 @@ llvm.func @minnum_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vect
llvm.return
}
+// CHECK-LABEL: @minimumnum_test
+llvm.func @minimumnum_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vector<8xf32>) {
+ // CHECK: call float @llvm.minimumnum.f32
+ "llvm.intr.minimumnum"(%arg0, %arg1) : (f32, f32) -> f32
+ // CHECK: call <8 x float> @llvm.minimumnum.v8f32
+ "llvm.intr.minimumnum"(%arg2, %arg3) : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+ llvm.return
+}
+
+// CHECK-LABEL: @maximumnum_test
+llvm.func @maximumnum_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vector<8xf32>) {
+ // CHECK: call float @llvm.maximumnum.f32
+ "llvm.intr.maximumnum"(%arg0, %arg1) : (f32, f32) -> f32
+ // CHECK: call <8 x float> @llvm.maximumnum.v8f32
+ "llvm.intr.maximumnum"(%arg2, %arg3) : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+ llvm.return
+}
+
// CHECK-LABEL: @smax_test
llvm.func @smax_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) {
// CHECK: call i32 @llvm.smax.i32
@@ -545,6 +563,10 @@ llvm.func @vector_reductions(%arg0: f32, %arg1: vector<8xf32>, %arg2: vector<8xi
llvm.intr.vector.reduce.fmaximum(%arg1) : (vector<8xf32>) -> f32
// CHECK: call float @llvm.vector.reduce.fminimum.v8f32
llvm.intr.vector.reduce.fminimum(%arg1) : (vector<8xf32>) -> f32
+ // CHECK: call float @llvm.vector.reduce.fminimumnum.v8f32
+ llvm.intr.vector.reduce.fminimumnum(%arg1) : (vector<8xf32>) -> f32
+ // CHECK: call float @llvm.vector.reduce.fmaximumnum.v8f32
+ llvm.intr.vector.reduce.fmaximumnum(%arg1) : (vector<8xf32>) -> f32
// CHECK: call i32 @llvm.vector.reduce.mul.v8i32
"llvm.intr.vector.reduce.mul"(%arg2) : (vector<8xi32>) -> i32
// CHECK: call i32 @llvm.vector.reduce.or.v8i32
More information about the Mlir-commits
mailing list