[Mlir-commits] [mlir] [mlir][vector] Fix mask neutral value for masked fmaximum/fminimum reductions (PR #222275)
lorenzo chelini
llvmlistbot at llvm.org
Thu Sep 10 02:18:14 PDT 2026
https://github.com/chelini updated https://github.com/llvm/llvm-project/pull/222275
>From 0b032c057712044025f1fe9cbe8ee572bd859f8c Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 09:53:25 +0200
Subject: [PATCH 1/4] [mlir][vector] Fix mask neutral value for masked
fmaximum/fminimum reductions
Masked <maximumf>/<minimumf> reductions were lowered by overwriting the
inactive lanes with a neutral value and running the unmasked reduction.
The neutral value for the maximum was APFloat::getSmallest(Negative),
which is the smallest value *by magnitude*, i.e. the negative subnormal
closest to zero rather than the bottom of the range. Any inactive lane
therefore won the reduction against every active lane below -1.4e-45:
vector.mask [1, 1, 1, 0] { vector.reduction <maximumf>, [-5, -5, -5, -9] }
returned -1.4e-45 instead of -5. The minimum used getLargest, which is
correct for finite inputs but returns FLT_MAX when an active lane is
+Inf.
llvm.vp.reduce.fmaximum/fminimum have been available since #91782 but
had no counterpart in the LLVM dialect. Add the two ops and drop the
workaround, lowering to the predicated intrinsics directly. Unlike
vp.reduce.fmax/fmin, they propagate NaN and order -0.0 below +0.0,
matching vector.reduction <maximumf>/<minimumf>.
This removes the neutral vector and the select, leaving only the neutral
*start value*, which has the same constraint: it must be an infinity, as
the largest finite value loses to an infinite active lane. Under ninf an
infinity may not be materialized, and need not be, since the flag rules
out the infinite lanes a finite neutral would lose to; the lowest and
largest finite values are used instead.
The existing tests only checked the emitted constant, so they recorded
the wrong value rather than catching it. Add an execution test covering
the inputs a too-small neutral loses to, and ninf coverage to the
conversion test.
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 2 +
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 125 ++++++++++--------
.../vector-reduction-to-llvm.mlir | 72 ++++++++--
.../CPU/reductions-masked-minmax-f32.mlir | 80 +++++++++++
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 6 +
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 8 ++
6 files changed, 233 insertions(+), 60 deletions(-)
create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index c02f9225ab159..eb46921319e88 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1850,6 +1850,8 @@ def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">;
def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">;
def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">;
def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">;
+def LLVM_VPReduceFMaximumOp : LLVM_VPReductionF<"fmaximum">;
+def LLVM_VPReduceFMinimumOp : LLVM_VPReductionF<"fminimum">;
def LLVM_VPMergeMinOp : LLVM_VPSelectBase<"merge">;
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index c1252b86cbcbb..81dcf00133a94 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -482,6 +482,10 @@ class ReductionNeutralSIntMax {};
class ReductionNeutralUIntMax {};
class ReductionNeutralFPMin {};
class ReductionNeutralFPMax {};
+class ReductionNeutralFPNegInf {};
+class ReductionNeutralFPPosInf {};
+class ReductionNeutralFPLowestFinite {};
+class ReductionNeutralFPLargestFinite {};
/// Create the reduction neutral zero value.
static Value createReductionNeutralValue(ReductionNeutralZero neutral,
@@ -581,6 +585,55 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
/*Negative=*/true)));
}
+/// Create the reduction neutral negative infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPNegInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral positive infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPPosInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
+/// Create the reduction neutral lowest finite value.
+static Value createReductionNeutralValue(ReductionNeutralFPLowestFinite neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral largest finite value.
+static Value
+createReductionNeutralValue(ReductionNeutralFPLargestFinite neutral,
+ ConversionPatternRewriter &rewriter, Location loc,
+ Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
/// Returns `accumulator` if it has a valid value. Otherwise, creates and
/// returns a new accumulator value using `ReductionNeutral`.
template <class ReductionNeutral>
@@ -690,52 +743,6 @@ static Value createFPReductionComparisonOpLowering(
return result;
}
-/// Reduction neutral classes for overloading
-class MaskNeutralFMaximum {};
-class MaskNeutralFMinimum {};
-
-/// Get the mask neutral floating point maximum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMaximum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true);
-}
-/// Get the mask neutral floating point minimum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMinimum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false);
-}
-
-/// Create the mask neutral floating point MLIR vector constant
-template <typename MaskNeutral>
-static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Type vectorType) {
- const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
- auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
- auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
- return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
-}
-
-/// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
-/// intrinsics. It is a workaround to overcome the lack of masked intrinsics for
-/// `fmaximum`/`fminimum`.
-/// More information: https://github.com/llvm/llvm-project/issues/64940
-template <class LLVMRedIntrinOp, class MaskNeutral>
-static Value
-lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Value vectorOperand, Value accumulator,
- Value mask, LLVM::FastmathFlagsAttr fmf) {
- const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
- rewriter, loc, llvmType, vectorOperand.getType());
- const Value selectedVectorByMask = LLVM::SelectOp::create(
- rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
- return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
- rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
-}
-
template <class LLVMRedIntrinOp, class ReductionNeutral>
static Value
lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
@@ -973,6 +980,8 @@ class MaskedReductionOpConversion
LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
reductionOp.getContext(),
convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
+ const bool noInfs =
+ LLVM::bitEnumContainsAny(fmf.getValue(), LLVM::FastmathFlags::ninf);
Value result;
switch (kind) {
@@ -1034,14 +1043,26 @@ class MaskedReductionOpConversion
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MAXIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum,
- MaskNeutralFMaximum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ // `ninf` promises no infinity reaches the reduction, so the neutral start
+ // value must stay finite.
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPLowestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPNegInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MINIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum,
- MaskNeutralFMinimum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPLargestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPPosInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
index 2a7bd816ced19..6ac19a0e5f4fb 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -168,10 +168,9 @@ func.func @masked_reduce_maximumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_maximumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<-1.401300e-45> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0xFF800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
@@ -183,10 +182,67 @@ func.func @masked_reduce_minimumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_minimumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<3.40282347E+38> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0x7F800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_maximumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+// `fast` is a group that includes `ninf`, so it selects the finite neutral.
+
+func.func @masked_reduce_maximumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
new file mode 100644
index 0000000000000..74ae2da875b91
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -0,0 +1,80 @@
+// RUN: mlir-opt %s -test-lower-to-llvm | \
+// RUN: mlir-runner -e entry -entry-point-result=void \
+// RUN: -shared-libs=%mlir_c_runner_utils | \
+// RUN: FileCheck %s
+
+func.func @entry() {
+ %inf = arith.constant 0x7F800000 : f32
+ %ninf = arith.constant 0xFF800000 : f32
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+
+ // Ordinary negative data. A neutral value anywhere above the active lanes,
+ // such as a negative subnormal, would be returned instead of the maximum.
+ %v0 = arith.constant dense<[-5.0, -7.0, -5.0, -9.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %0 : f32
+ // CHECK: -5
+
+ // Mirror image for the minimum.
+ %v1 = arith.constant dense<[5.0, 7.0, 5.0, 9.0]> : vector<4xf32>
+ %1 = vector.mask %mask {
+ vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %1 : f32
+ // CHECK: 5
+
+ // Infinite active lanes: the largest finite value would win over them.
+ %v2 = vector.broadcast %ninf : f32 to vector<4xf32>
+ %2 = vector.mask %mask {
+ vector.reduction <maximumf>, %v2 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %2 : f32
+ // CHECK: -inf
+
+ %v3 = vector.broadcast %inf : f32 to vector<4xf32>
+ %3 = vector.mask %mask {
+ vector.reduction <minimumf>, %v3 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %3 : f32
+ // CHECK: inf
+
+ // No active lane: the result is the identity of the reduction.
+ %nomask = arith.constant dense<false> : vector<4xi1>
+ %4 = vector.mask %nomask {
+ vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %4 : f32
+ // CHECK: -inf
+
+ %5 = vector.mask %nomask {
+ vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %5 : f32
+ // CHECK: inf
+
+ // An accumulator combines with the masked reduction.
+ %acc = arith.constant -3.0 : f32
+ %6 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0, %acc : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %6 : f32
+ // CHECK: -3
+
+ // Under `ninf` the neutral value is the largest finite value instead. It is
+ // still neutral for the finite inputs the flag restricts the reduction to.
+ %7 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0 fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %7 : f32
+ // CHECK: -5
+
+ %8 = vector.mask %mask {
+ vector.reduction <minimumf>, %v1 fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %8 : f32
+ // CHECK: 5
+
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index eb0eaaf43b1f8..694d21c2f9f53 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -1145,6 +1145,10 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa
%45 = call float @llvm.vp.reduce.fmax.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: "llvm.intr.vp.reduce.fmin"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
%46 = call float @llvm.vp.reduce.fmin.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fmaximum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fmaximum = call float @llvm.vp.reduce.fmaximum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fminimum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fminimum = call float @llvm.vp.reduce.fminimum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: llvm.select %{{.*}}, %{{.*}}, %{{.*}} : vector<8xi1>, vector<8xi32>
%47 = call <8 x i32> @llvm.vp.select.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12)
; CHECK: "llvm.intr.vp.merge"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi1>, vector<8xi32>, vector<8xi32>, i32) -> vector<8xi32>
@@ -1969,6 +1973,8 @@ declare i32 @llvm.vp.reduce.umin.v8i32(i32, <8 x i32>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index d16e54578f157..5268a2772a506 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -1130,6 +1130,12 @@ llvm.func @vector_predication_intrinsics(%A: vector<8xi32>, %B: vector<8xi32>,
// CHECK: call float @llvm.vp.reduce.fmin.v8f32
"llvm.intr.vp.reduce.fmin" (%f, %C, %mask, %evl) :
(f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fmaximum.v8f32
+ "llvm.intr.vp.reduce.fmaximum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fminimum.v8f32
+ "llvm.intr.vp.reduce.fminimum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
// CHECK: call <8 x i32> @llvm.vp.merge.v8i32
"llvm.intr.vp.merge" (%mask, %A, %B, %evl) :
@@ -2128,6 +2134,8 @@ llvm.func @vector_scmp(%a: vector<4 x i32>, %b: vector<4 x i32>) -> vector<4 x i
// CHECK-DAG: declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
// CHECK-DAG: declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr captures(none), i32, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr captures(none), i32, <8 x i1>, i32)
>From 8b6a407fdf4cfd4269fef8a9addd491678f8e8c3 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 12:52:48 +0200
Subject: [PATCH 2/4] Address review: state the neutral requirement instead of
a counterfactual
---
.../CPU/reductions-masked-minmax-f32.mlir | 176 ++++++++++++------
1 file changed, 124 insertions(+), 52 deletions(-)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
index 74ae2da875b91..797d04f786c1f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -3,78 +3,150 @@
// RUN: -shared-libs=%mlir_c_runner_utils | \
// RUN: FileCheck %s
-func.func @entry() {
- %inf = arith.constant 0x7F800000 : f32
- %ninf = arith.constant 0xFF800000 : f32
+func.func @maximumf_finite() {
+ // The neutral (-inf) must order below every active lane. The masked-off lane is the
+ // largest.
+ // max(<-inf>, -5, -7, -5) = -5
%mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
-
- // Ordinary negative data. A neutral value anywhere above the active lanes,
- // such as a negative subnormal, would be returned instead of the maximum.
- %v0 = arith.constant dense<[-5.0, -7.0, -5.0, -9.0]> : vector<4xf32>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
%0 = vector.mask %mask {
- vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
+ vector.print str "maximumf_finite\n"
vector.print %0 : f32
- // CHECK: -5
+ return
+}
+// CHECK-LABEL: maximumf_finite
+// CHECK-NEXT: -5
- // Mirror image for the minimum.
- %v1 = arith.constant dense<[5.0, 7.0, 5.0, 9.0]> : vector<4xf32>
- %1 = vector.mask %mask {
- vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+func.func @minimumf_finite() {
+ // min(<+inf>, 5, 7, 5) = 5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %1 : f32
- // CHECK: 5
+ vector.print str "minimumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_finite
+// CHECK-NEXT: 5
- // Infinite active lanes: the largest finite value would win over them.
- %v2 = vector.broadcast %ninf : f32 to vector<4xf32>
- %2 = vector.mask %mask {
- vector.reduction <maximumf>, %v2 : vector<4xf32> into f32
+func.func @maximumf_inf() {
+ // The neutral must be an infinity: a finite one wins over -inf lanes.
+ // max(<-inf>, -inf, -inf, -inf) = -inf
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %ninf = arith.constant 0xFF800000 : f32
+ %v = vector.broadcast %ninf : f32 to vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %2 : f32
- // CHECK: -inf
+ vector.print str "maximumf_inf\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_inf
+// CHECK-NEXT: -inf
- %v3 = vector.broadcast %inf : f32 to vector<4xf32>
- %3 = vector.mask %mask {
- vector.reduction <minimumf>, %v3 : vector<4xf32> into f32
+func.func @minimumf_inf() {
+ // min(<+inf>, +inf, +inf, +inf) = +inf
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %inf = arith.constant 0x7F800000 : f32
+ %v = vector.broadcast %inf : f32 to vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %3 : f32
- // CHECK: inf
+ vector.print str "minimumf_inf\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_inf
+// CHECK-NEXT: inf
- // No active lane: the result is the identity of the reduction.
- %nomask = arith.constant dense<false> : vector<4xi1>
- %4 = vector.mask %nomask {
- vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+func.func @maximumf_no_active_lane() {
+ // With no active lane the neutral wins.
+ // max(<-inf>) = -inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %4 : f32
- // CHECK: -inf
+ vector.print str "maximumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_no_active_lane
+// CHECK-NEXT: -inf
- %5 = vector.mask %nomask {
- vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+func.func @minimumf_no_active_lane() {
+ // min(<+inf>) = +inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %5 : f32
- // CHECK: inf
+ vector.print str "minimumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_no_active_lane
+// CHECK-NEXT: inf
- // An accumulator combines with the masked reduction.
- %acc = arith.constant -3.0 : f32
- %6 = vector.mask %mask {
- vector.reduction <maximumf>, %v0, %acc : vector<4xf32> into f32
+func.func @maximumf_ninf_no_active_lane() {
+ // Under `ninf` the neutral is -FLT_MAX instead of -inf.
+ // max(<-FLT_MAX>) = -FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v fastmath<ninf> : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %6 : f32
- // CHECK: -3
+ vector.print str "maximumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_ninf_no_active_lane
+// CHECK-NEXT: -3.40282e+38
- // Under `ninf` the neutral value is the largest finite value instead. It is
- // still neutral for the finite inputs the flag restricts the reduction to.
- %7 = vector.mask %mask {
- vector.reduction <maximumf>, %v0 fastmath<ninf> : vector<4xf32> into f32
+func.func @minimumf_ninf_no_active_lane() {
+ // min(<+FLT_MAX>) = +FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v fastmath<ninf> : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %7 : f32
- // CHECK: -5
+ vector.print str "minimumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_ninf_no_active_lane
+// CHECK-NEXT: 3.40282e+38
- %8 = vector.mask %mask {
- vector.reduction <minimumf>, %v1 fastmath<ninf> : vector<4xf32> into f32
+func.func @maximumf_acc() {
+ // An accumulator takes the place of the neutral.
+ // max(<-3>, -5, -7, -5) = -3
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %acc = arith.constant -3.0 : f32
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v, %acc : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %8 : f32
- // CHECK: 5
+ vector.print str "maximumf_acc\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_acc
+// CHECK-NEXT: -3
+func.func @entry() {
+ call @maximumf_finite() : () -> ()
+ call @minimumf_finite() : () -> ()
+ call @maximumf_inf() : () -> ()
+ call @minimumf_inf() : () -> ()
+ call @maximumf_no_active_lane() : () -> ()
+ call @minimumf_no_active_lane() : () -> ()
+ call @maximumf_ninf_no_active_lane() : () -> ()
+ call @minimumf_ninf_no_active_lane() : () -> ()
+ call @maximumf_acc() : () -> ()
return
}
>From cc4672ba671432ed733b89fd143896146883ea82 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 13:15:44 +0200
Subject: [PATCH 3/4] Remove confusing tests
---
.../CPU/reductions-masked-minmax-f32.mlir | 37 +------------------
1 file changed, 2 insertions(+), 35 deletions(-)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
index 797d04f786c1f..4dcbac06cb205 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -4,8 +4,8 @@
// RUN: FileCheck %s
func.func @maximumf_finite() {
- // The neutral (-inf) must order below every active lane. The masked-off lane is the
- // largest.
+ // The neutral (-inf) must order below every active lane. The masked-off
+ // lane is the largest.
// max(<-inf>, -5, -7, -5) = -5
%mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
%v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
@@ -33,37 +33,6 @@ func.func @minimumf_finite() {
// CHECK-LABEL: minimumf_finite
// CHECK-NEXT: 5
-func.func @maximumf_inf() {
- // The neutral must be an infinity: a finite one wins over -inf lanes.
- // max(<-inf>, -inf, -inf, -inf) = -inf
- %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
- %ninf = arith.constant 0xFF800000 : f32
- %v = vector.broadcast %ninf : f32 to vector<4xf32>
- %0 = vector.mask %mask {
- vector.reduction <maximumf>, %v : vector<4xf32> into f32
- } : vector<4xi1> -> f32
- vector.print str "maximumf_inf\n"
- vector.print %0 : f32
- return
-}
-// CHECK-LABEL: maximumf_inf
-// CHECK-NEXT: -inf
-
-func.func @minimumf_inf() {
- // min(<+inf>, +inf, +inf, +inf) = +inf
- %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
- %inf = arith.constant 0x7F800000 : f32
- %v = vector.broadcast %inf : f32 to vector<4xf32>
- %0 = vector.mask %mask {
- vector.reduction <minimumf>, %v : vector<4xf32> into f32
- } : vector<4xi1> -> f32
- vector.print str "minimumf_inf\n"
- vector.print %0 : f32
- return
-}
-// CHECK-LABEL: minimumf_inf
-// CHECK-NEXT: inf
-
func.func @maximumf_no_active_lane() {
// With no active lane the neutral wins.
// max(<-inf>) = -inf
@@ -141,8 +110,6 @@ func.func @maximumf_acc() {
func.func @entry() {
call @maximumf_finite() : () -> ()
call @minimumf_finite() : () -> ()
- call @maximumf_inf() : () -> ()
- call @minimumf_inf() : () -> ()
call @maximumf_no_active_lane() : () -> ()
call @minimumf_no_active_lane() : () -> ()
call @maximumf_ninf_no_active_lane() : () -> ()
>From d3a13cccb115af449ddbf93853278dcafa2f5fd2 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Thu, 10 Sep 2026 11:17:38 +0200
Subject: [PATCH 4/4] address comments
---
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 21 ++++++++++---------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 81dcf00133a94..45efc531e7ad6 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -480,8 +480,8 @@ class ReductionNeutralSIntMin {};
class ReductionNeutralUIntMin {};
class ReductionNeutralSIntMax {};
class ReductionNeutralUIntMax {};
-class ReductionNeutralFPMin {};
-class ReductionNeutralFPMax {};
+class ReductionNeutralFPQNaN {};
+class ReductionNeutralFPNegQNaN {};
class ReductionNeutralFPNegInf {};
class ReductionNeutralFPPosInf {};
class ReductionNeutralFPLowestFinite {};
@@ -561,8 +561,8 @@ static Value createReductionNeutralValue(ReductionNeutralUIntMax neutral,
llvmType.getIntOrFloatBitWidth())));
}
-/// Create the reduction neutral fp minimum value.
-static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
+/// Create the reduction neutral quiet NaN value.
+static Value createReductionNeutralValue(ReductionNeutralFPQNaN neutral,
ConversionPatternRewriter &rewriter,
Location loc, Type llvmType) {
auto floatType = cast<FloatType>(llvmType);
@@ -573,8 +573,8 @@ static Value createReductionNeutralValue(ReductionNeutralFPMin neutral,
/*Negative=*/false)));
}
-/// Create the reduction neutral fp maximum value.
-static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
+/// Create the reduction neutral negative quiet NaN value.
+static Value createReductionNeutralValue(ReductionNeutralFPNegQNaN neutral,
ConversionPatternRewriter &rewriter,
Location loc, Type llvmType) {
auto floatType = cast<FloatType>(llvmType);
@@ -1033,13 +1033,14 @@ class MaskedReductionOpConversion
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case vector::CombiningKind::MINNUMF:
- result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
- ReductionNeutralFPMax>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask());
+ result =
+ lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMinOp,
+ ReductionNeutralFPNegQNaN>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case vector::CombiningKind::MAXNUMF:
result = lowerPredicatedReductionWithStartValue<LLVM::VPReduceFMaxOp,
- ReductionNeutralFPMin>(
+ ReductionNeutralFPQNaN>(
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MAXIMUMF:
More information about the Mlir-commits
mailing list