[Mlir-commits] [mlir] b3eab3a - [mlir][vector] Fix mask neutral value for masked fmaximum/fminimum reductions (#222275)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Thu Sep 10 05:11:27 PDT 2026
Author: lorenzo chelini
Date: 2026-09-10T14:11:22+02:00
New Revision: b3eab3a932118c3a462741000d2118428f5bd268
URL: https://github.com/llvm/llvm-project/commit/b3eab3a932118c3a462741000d2118428f5bd268
DIFF: https://github.com/llvm/llvm-project/commit/b3eab3a932118c3a462741000d2118428f5bd268.diff
LOG: [mlir][vector] Fix mask neutral value for masked fmaximum/fminimum reductions (#222275)
Masked maximumf/minimumf reductions were lowered by replacing inactive
lanes with a neutral value before applying an unmasked reduction. The
maximumf neutral was incorrectly set to the negative subnormal closest
to zero, causing it to beat any negative active value. Similarly,
minimumf used FLT_MAX, which loses to an active +Inf.
This change adds LLVM dialect support for
llvm.vp.reduce.fmaximum/fminimum and lowers masked reductions directly
to these predicated intrinsics.
Added:
mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
Modified:
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
mlir/test/Target/LLVMIR/Import/intrinsic.ll
mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index c02f9225ab159..eb46921319e88 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1850,6 +1850,8 @@ def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">;
def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">;
def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">;
def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">;
+def LLVM_VPReduceFMaximumOp : LLVM_VPReductionF<"fmaximum">;
+def LLVM_VPReduceFMinimumOp : LLVM_VPReductionF<"fminimum">;
def LLVM_VPMergeMinOp : LLVM_VPSelectBase<"merge">;
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index c1252b86cbcbb..45efc531e7ad6 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -480,8 +480,12 @@ class ReductionNeutralSIntMin {};
class ReductionNeutralUIntMin {};
class ReductionNeutralSIntMax {};
class ReductionNeutralUIntMax {};
-class ReductionNeutralFPMin {};
-class ReductionNeutralFPMax {};
+class ReductionNeutralFPQNaN {};
+class ReductionNeutralFPNegQNaN {};
+class ReductionNeutralFPNegInf {};
+class ReductionNeutralFPPosInf {};
+class ReductionNeutralFPLowestFinite {};
+class ReductionNeutralFPLargestFinite {};
/// Create the reduction neutral zero value.
static Value createReductionNeutralValue(ReductionNeutralZero neutral,
@@ -557,8 +561,8 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
llvmType.getIntOrFloatBitWidth())));
}
-/// Create the reduction neutral fp minimum value.
-static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
+/// Create the reduction neutral quiet NaN value.
+static Value createReductionNeutralValue(ReductionNeutralFPQNaN neutral,
ConversionPatternRewriter &rewriter,
Location loc, Type llvmType) {
auto floatType = cast<FloatType>(llvmType);
@@ -569,8 +573,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
/*Negative=*/false)));
}
-/// Create the reduction neutral fp maximum value.
-static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
+/// Create the reduction neutral negative quiet NaN value.
+static Value createReductionNeutralValue(ReductionNeutralFPNegQNaN neutral,
ConversionPatternRewriter &rewriter,
Location loc, Type llvmType) {
auto floatType = cast<FloatType>(llvmType);
@@ -581,6 +585,55 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
/*Negative=*/true)));
}
+/// Create the reduction neutral negative infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPNegInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral positive infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPPosInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
+/// Create the reduction neutral lowest finite value.
+static Value createReductionNeutralValue(ReductionNeutralFPLowestFinite neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral largest finite value.
+static Value
+createReductionNeutralValue(ReductionNeutralFPLargestFinite neutral,
+ ConversionPatternRewriter &rewriter, Location loc,
+ Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
/// Returns `accumulator` if it has a valid value. Otherwise, creates and
/// returns a new accumulator value using `ReductionNeutral`.
template <class ReductionNeutral>
@@ -690,52 +743,6 @@ static Value createFPReductionComparisonOpLowering(
return result;
}
-/// Reduction neutral classes for overloading
-class MaskNeutralFMaximum {};
-class MaskNeutralFMinimum {};
-
-/// Get the mask neutral floating point maximum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMaximum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true);
-}
-/// Get the mask neutral floating point minimum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMinimum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false);
-}
-
-/// Create the mask neutral floating point MLIR vector constant
-template <typename MaskNeutral>
-static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Type vectorType) {
- const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
- auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
- auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
- return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
-}
-
-/// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
-/// intrinsics. It is a workaround to overcome the lack of masked intrinsics for
-/// `fmaximum`/`fminimum`.
-/// More information: https://github.com/llvm/llvm-project/issues/64940
-template <class LLVMRedIntrinOp, class MaskNeutral>
-static Value
-lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Value vectorOperand, Value accumulator,
- Value mask, LLVM::FastmathFlagsAttr fmf) {
- const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
- rewriter, loc, llvmType, vectorOperand.getType());
- const Value selectedVectorByMask = LLVM::SelectOp::create(
- rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
- return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
- rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
-}
-
template <class LLVMRedIntrinOp, class ReductionNeutral>
static Value
lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
@@ -973,6 +980,8 @@ class MaskedReductionOpConversion
LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
reductionOp.getContext(),
convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
+ const bool noInfs =
+ LLVM::bitEnumContainsAny(fmf.getValue(), LLVM::FastmathFlags::ninf);
Value result;
switch (kind) {
@@ -1024,24 +1033,37 @@ class MaskedReductionOpConversion
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case vector::CombiningKind::MINNUMF:
- result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
- ReductionNeutralFPMax>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask());
+ result =
+ lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
+ ReductionNeutralFPNegQNaN>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case vector::CombiningKind::MAXNUMF:
result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMaxOp,
- ReductionNeutralFPMin>(
+ ReductionNeutralFPQNaN>(
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MAXIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum,
- MaskNeutralFMaximum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ // `ninf` promises no infinity reaches the reduction, so the neutral start
+ // value must stay finite.
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPLowestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPNegInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MINIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum,
- MaskNeutralFMinimum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPLargestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPPosInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
index 2a7bd816ced19..6ac19a0e5f4fb 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -168,10 +168,9 @@ func.func @masked_reduce_maximumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_maximumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<-1.401300e-45> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0xFF800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
@@ -183,10 +182,67 @@ func.func @masked_reduce_minimumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_minimumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<3.40282347E+38> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0x7F800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_maximumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+// `fast` is a group that includes `ninf`, so it selects the finite neutral.
+
+func.func @masked_reduce_maximumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
new file mode 100644
index 0000000000000..4dcbac06cb205
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -0,0 +1,119 @@
+// RUN: mlir-opt %s -test-lower-to-llvm | \
+// RUN: mlir-runner -e entry -entry-point-result=void \
+// RUN: -shared-libs=%mlir_c_runner_utils | \
+// RUN: FileCheck %s
+
+func.func @maximumf_finite() {
+ // The neutral (-inf) must order below every active lane. The masked-off
+ // lane is the largest.
+ // max(<-inf>, -5, -7, -5) = -5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_finite
+// CHECK-NEXT: -5
+
+func.func @minimumf_finite() {
+ // min(<+inf>, 5, 7, 5) = 5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_finite
+// CHECK-NEXT: 5
+
+func.func @maximumf_no_active_lane() {
+ // With no active lane the neutral wins.
+ // max(<-inf>) = -inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_no_active_lane
+// CHECK-NEXT: -inf
+
+func.func @minimumf_no_active_lane() {
+ // min(<+inf>) = +inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_no_active_lane
+// CHECK-NEXT: inf
+
+func.func @maximumf_ninf_no_active_lane() {
+ // Under `ninf` the neutral is -FLT_MAX instead of -inf.
+ // max(<-FLT_MAX>) = -FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_ninf_no_active_lane
+// CHECK-NEXT: -3.40282e+38
+
+func.func @minimumf_ninf_no_active_lane() {
+ // min(<+FLT_MAX>) = +FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "minimumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_ninf_no_active_lane
+// CHECK-NEXT: 3.40282e+38
+
+func.func @maximumf_acc() {
+ // An accumulator takes the place of the neutral.
+ // max(<-3>, -5, -7, -5) = -3
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %acc = arith.constant -3.0 : f32
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v, %acc : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print str "maximumf_acc\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_acc
+// CHECK-NEXT: -3
+
+func.func @entry() {
+ call @maximumf_finite() : () -> ()
+ call @minimumf_finite() : () -> ()
+ call @maximumf_no_active_lane() : () -> ()
+ call @minimumf_no_active_lane() : () -> ()
+ call @maximumf_ninf_no_active_lane() : () -> ()
+ call @minimumf_ninf_no_active_lane() : () -> ()
+ call @maximumf_acc() : () -> ()
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index eb0eaaf43b1f8..694d21c2f9f53 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -1145,6 +1145,10 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa
%45 = call float @llvm.vp.reduce.fmax.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: "llvm.intr.vp.reduce.fmin"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
%46 = call float @llvm.vp.reduce.fmin.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fmaximum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fmaximum = call float @llvm.vp.reduce.fmaximum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fminimum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fminimum = call float @llvm.vp.reduce.fminimum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: llvm.select %{{.*}}, %{{.*}}, %{{.*}} : vector<8xi1>, vector<8xi32>
%47 = call <8 x i32> @llvm.vp.select.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12)
; CHECK: "llvm.intr.vp.merge"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi1>, vector<8xi32>, vector<8xi32>, i32) -> vector<8xi32>
@@ -1969,6 +1973,8 @@ declare i32 @llvm.vp.reduce.umin.v8i32(i32, <8 x i32>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index d16e54578f157..5268a2772a506 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -1130,6 +1130,12 @@ llvm.func @vector_predication_intrinsics(%A: vector<8xi32>, %B: vector<8xi32>,
// CHECK: call float @llvm.vp.reduce.fmin.v8f32
"llvm.intr.vp.reduce.fmin" (%f, %C, %mask, %evl) :
(f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fmaximum.v8f32
+ "llvm.intr.vp.reduce.fmaximum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fminimum.v8f32
+ "llvm.intr.vp.reduce.fminimum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
// CHECK: call <8 x i32> @llvm.vp.merge.v8i32
"llvm.intr.vp.merge" (%mask, %A, %B, %evl) :
@@ -2128,6 +2134,8 @@ llvm.func @vector_scmp(%a: vector<4 x i32>, %b: vector<4 x i32>) -> vector<4 x i
// CHECK-DAG: declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
// CHECK-DAG: declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr captures(none), i32, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr captures(none), i32, <8 x i1>, i32)
More information about the Mlir-commits
mailing list