[Mlir-commits] [mlir] [mlir][vector] Fix mask neutral value for masked fmaximum/fminimum reductions (PR #222275)
lorenzo chelini
llvmlistbot at llvm.org
Wed Sep 9 04:16:01 PDT 2026
https://github.com/chelini updated https://github.com/llvm/llvm-project/pull/222275
>From b2fcce8fcace0b0b6a971b7e1edc48d54fb3fc31 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 09:53:25 +0200
Subject: [PATCH 1/3] [mlir][vector] Fix mask neutral value for masked
fmaximum/fminimum reductions
Masked <maximumf>/<minimumf> reductions were lowered by overwriting the
inactive lanes with a neutral value and running the unmasked reduction.
The neutral value for the maximum was APFloat::getSmallest(Negative),
which is the smallest value *by magnitude*, i.e. the negative subnormal
closest to zero rather than the bottom of the range. Any inactive lane
therefore won the reduction against every active lane below -1.4e-45:
vector.mask [1, 1, 1, 0] { vector.reduction <maximumf>, [-5, -5, -5, -9] }
returned -1.4e-45 instead of -5. The minimum used getLargest, which is
correct for finite inputs but returns FLT_MAX when an active lane is
+Inf.
llvm.vp.reduce.fmaximum/fminimum have been available since #91782 but
had no counterpart in the LLVM dialect. Add the two ops and drop the
workaround, lowering to the predicated intrinsics directly. Unlike
vp.reduce.fmax/fmin, they propagate NaN and order -0.0 below +0.0,
matching vector.reduction <maximumf>/<minimumf>.
This removes the neutral vector and the select, leaving only the neutral
*start value*, which has the same constraint: it must be an infinity, as
the largest finite value loses to an infinite active lane. Under ninf an
infinity may not be materialized, and need not be, since the flag rules
out the infinite lanes a finite neutral would lose to; the lowest and
largest finite values are used instead.
The existing tests only checked the emitted constant, so they recorded
the wrong value rather than catching it. Add an execution test covering
the inputs a too-small neutral loses to, and ninf coverage to the
conversion test.
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 2 +
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 125 ++++++++++--------
.../vector-reduction-to-llvm.mlir | 72 ++++++++--
.../CPU/reductions-masked-minmax-f32.mlir | 80 +++++++++++
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 6 +
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 8 ++
6 files changed, 233 insertions(+), 60 deletions(-)
create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index f40f3f5ebda93..d494bd7184a16 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1702,6 +1702,8 @@ def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">;
def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">;
def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">;
def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">;
+def LLVM_VPReduceFMaximumOp : LLVM_VPReductionF<"fmaximum">;
+def LLVM_VPReduceFMinimumOp : LLVM_VPReductionF<"fminimum">;
def LLVM_VPMergeMinOp : LLVM_VPSelectBase<"merge">;
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 152c45b695cee..53f28d5ff80e0 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -483,6 +483,10 @@ class ReductionNeutralSIntMax {};
class ReductionNeutralUIntMax {};
class ReductionNeutralFPMin {};
class ReductionNeutralFPMax {};
+class ReductionNeutralFPNegInf {};
+class ReductionNeutralFPPosInf {};
+class ReductionNeutralFPLowestFinite {};
+class ReductionNeutralFPLargestFinite {};
/// Create the reduction neutral zero value.
static Value createReductionNeutralValue(ReductionNeutralZero neutral,
@@ -582,6 +586,55 @@ static Value createReductionNeutralValue(ReductionNeutralFPMax neutral,
/*Negative=*/true)));
}
+/// Create the reduction neutral negative infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPNegInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral positive infinity value.
+static Value createReductionNeutralValue(ReductionNeutralFPPosInf neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(llvmType,
+ llvm::APFloat::getInf(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
+/// Create the reduction neutral lowest finite value.
+static Value createReductionNeutralValue(ReductionNeutralFPLowestFinite neutral,
+ ConversionPatternRewriter &rewriter,
+ Location loc, Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/true)));
+}
+
+/// Create the reduction neutral largest finite value.
+static Value
+createReductionNeutralValue(ReductionNeutralFPLargestFinite neutral,
+ ConversionPatternRewriter &rewriter, Location loc,
+ Type llvmType) {
+ auto floatType = cast<FloatType>(llvmType);
+ return LLVM::ConstantOp::create(
+ rewriter, loc, llvmType,
+ rewriter.getFloatAttr(
+ llvmType, llvm::APFloat::getLargest(floatType.getFloatSemantics(),
+ /*Negative=*/false)));
+}
+
/// Returns `accumulator` if it has a valid value. Otherwise, creates and
/// returns a new accumulator value using `ReductionNeutral`.
template <class ReductionNeutral>
@@ -691,52 +744,6 @@ static Value createFPReductionComparisonOpLowering(
return result;
}
-/// Reduction neutral classes for overloading
-class MaskNeutralFMaximum {};
-class MaskNeutralFMinimum {};
-
-/// Get the mask neutral floating point maximum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMaximum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getSmallest(floatSemantics, /*Negative=*/true);
-}
-/// Get the mask neutral floating point minimum value
-static llvm::APFloat
-getMaskNeutralValue(MaskNeutralFMinimum,
- const llvm::fltSemantics &floatSemantics) {
- return llvm::APFloat::getLargest(floatSemantics, /*Negative=*/false);
-}
-
-/// Create the mask neutral floating point MLIR vector constant
-template <typename MaskNeutral>
-static Value createMaskNeutralValue(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Type vectorType) {
- const auto &floatSemantics = cast<FloatType>(llvmType).getFloatSemantics();
- auto value = getMaskNeutralValue(MaskNeutral{}, floatSemantics);
- auto denseValue = DenseElementsAttr::get(cast<ShapedType>(vectorType), value);
- return LLVM::ConstantOp::create(rewriter, loc, vectorType, denseValue);
-}
-
-/// Lowers masked `fmaximum` and `fminimum` reductions using the non-masked
-/// intrinsics. It is a workaround to overcome the lack of masked intrinsics for
-/// `fmaximum`/`fminimum`.
-/// More information: https://github.com/llvm/llvm-project/issues/64940
-template <class LLVMRedIntrinOp, class MaskNeutral>
-static Value
-lowerMaskedReductionWithRegular(ConversionPatternRewriter &rewriter,
- Location loc, Type llvmType,
- Value vectorOperand, Value accumulator,
- Value mask, LLVM::FastmathFlagsAttr fmf) {
- const Value vectorMaskNeutral = createMaskNeutralValue<MaskNeutral>(
- rewriter, loc, llvmType, vectorOperand.getType());
- const Value selectedVectorByMask = LLVM::SelectOp::create(
- rewriter, loc, mask, vectorOperand, vectorMaskNeutral);
- return createFPReductionComparisonOpLowering<LLVMRedIntrinOp>(
- rewriter, loc, llvmType, selectedVectorByMask, accumulator, fmf);
-}
-
template <class LLVMRedIntrinOp, class ReductionNeutral>
static Value
lowerReductionWithStartValue(ConversionPatternRewriter &rewriter, Location loc,
@@ -974,6 +981,8 @@ class MaskedReductionOpConversion
LLVM::FastmathFlagsAttr fmf = LLVM::FastmathFlagsAttr::get(
reductionOp.getContext(),
convertArithFastMathFlagsToLLVM(fMFAttr.getValue()));
+ const bool noInfs =
+ LLVM::bitEnumContainsAny(fmf.getValue(), LLVM::FastmathFlags::ninf);
Value result;
switch (kind) {
@@ -1035,14 +1044,26 @@ class MaskedReductionOpConversion
rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MAXIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fmaximum,
- MaskNeutralFMaximum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ // `ninf` promises no infinity reaches the reduction, so the neutral start
+ // value must stay finite.
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPLowestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMaximumOp, ReductionNeutralFPNegInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
case CombiningKind::MINIMUMF:
- result = lowerMaskedReductionWithRegular<LLVM::vector_reduce_fminimum,
- MaskNeutralFMinimum>(
- rewriter, loc, llvmType, operand, acc, maskOp.getMask(), fmf);
+ result =
+ noInfs
+ ? lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPLargestFinite>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask())
+ : lowerPredicatedReductionWithStartValue<
+ LLVM::VPReduceFMinimumOp, ReductionNeutralFPPosInf>(
+ rewriter, loc, llvmType, operand, acc, maskOp.getMask());
break;
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
index c7d9e22fb2423..434dd87a7293a 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-reduction-to-llvm.mlir
@@ -169,10 +169,9 @@ func.func @masked_reduce_maximumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_maximumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<-1.401300e-45> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fmaximum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0xFF800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
@@ -184,10 +183,67 @@ func.func @masked_reduce_minimumf_f32(%arg0: vector<16xf32>, %mask : vector<16xi
// CHECK-LABEL: func.func @masked_reduce_minimumf_f32(
// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
-// CHECK: %[[MASK_NEUTRAL:.*]] = llvm.mlir.constant(dense<3.40282347E+38> : vector<16xf32>) : vector<16xf32>
-// CHECK: %[[MASKED:.*]] = llvm.select %[[MASK]], %[[INPUT]], %[[MASK_NEUTRAL]] : vector<16xi1>, vector<16xf32>
-// CHECK: %[[RESULT:.*]] = llvm.intr.vector.reduce.fminimum(%[[MASKED]]) : (vector<16xf32>) -> f32
-// CHECK: return %[[RESULT]]
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(0x7F800000 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_maximumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_ninf(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<ninf> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_ninf(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+// `fast` is a group that includes `ninf`, so it selects the finite neutral.
+
+func.func @masked_reduce_maximumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <maximumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_maximumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(-3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fmaximum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
+
+// -----
+
+func.func @masked_reduce_minimumf_f32_fast(%arg0: vector<16xf32>, %mask : vector<16xi1>) -> f32 {
+ %0 = vector.mask %mask { vector.reduction <minimumf>, %arg0 fastmath<fast> : vector<16xf32> into f32 } : vector<16xi1> -> f32
+ return %0 : f32
+}
+
+// CHECK-LABEL: func.func @masked_reduce_minimumf_f32_fast(
+// CHECK-SAME: %[[INPUT:.*]]: vector<16xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>) -> f32 {
+// CHECK: %[[NEUTRAL:.*]] = llvm.mlir.constant(3.40282347E+38 : f32) : f32
+// CHECK: %[[VL:.*]] = llvm.mlir.constant(16 : i32) : i32
+// CHECK: "llvm.intr.vp.reduce.fminimum"(%[[NEUTRAL]], %[[INPUT]], %[[MASK]], %[[VL]]) : (f32, vector<16xf32>, vector<16xi1>, i32) -> f32
// -----
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
new file mode 100644
index 0000000000000..74ae2da875b91
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -0,0 +1,80 @@
+// RUN: mlir-opt %s -test-lower-to-llvm | \
+// RUN: mlir-runner -e entry -entry-point-result=void \
+// RUN: -shared-libs=%mlir_c_runner_utils | \
+// RUN: FileCheck %s
+
+func.func @entry() {
+ %inf = arith.constant 0x7F800000 : f32
+ %ninf = arith.constant 0xFF800000 : f32
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+
+ // Ordinary negative data. A neutral value anywhere above the active lanes,
+ // such as a negative subnormal, would be returned instead of the maximum.
+ %v0 = arith.constant dense<[-5.0, -7.0, -5.0, -9.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %0 : f32
+ // CHECK: -5
+
+ // Mirror image for the minimum.
+ %v1 = arith.constant dense<[5.0, 7.0, 5.0, 9.0]> : vector<4xf32>
+ %1 = vector.mask %mask {
+ vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %1 : f32
+ // CHECK: 5
+
+ // Infinite active lanes: the largest finite value would win over them.
+ %v2 = vector.broadcast %ninf : f32 to vector<4xf32>
+ %2 = vector.mask %mask {
+ vector.reduction <maximumf>, %v2 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %2 : f32
+ // CHECK: -inf
+
+ %v3 = vector.broadcast %inf : f32 to vector<4xf32>
+ %3 = vector.mask %mask {
+ vector.reduction <minimumf>, %v3 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %3 : f32
+ // CHECK: inf
+
+ // No active lane: the result is the identity of the reduction.
+ %nomask = arith.constant dense<false> : vector<4xi1>
+ %4 = vector.mask %nomask {
+ vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %4 : f32
+ // CHECK: -inf
+
+ %5 = vector.mask %nomask {
+ vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %5 : f32
+ // CHECK: inf
+
+ // An accumulator combines with the masked reduction.
+ %acc = arith.constant -3.0 : f32
+ %6 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0, %acc : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %6 : f32
+ // CHECK: -3
+
+ // Under `ninf` the neutral value is the largest finite value instead. It is
+ // still neutral for the finite inputs the flag restricts the reduction to.
+ %7 = vector.mask %mask {
+ vector.reduction <maximumf>, %v0 fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %7 : f32
+ // CHECK: -5
+
+ %8 = vector.mask %mask {
+ vector.reduction <minimumf>, %v1 fastmath<ninf> : vector<4xf32> into f32
+ } : vector<4xi1> -> f32
+ vector.print %8 : f32
+ // CHECK: 5
+
+ return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index aaebaa2f46d1c..f8d67f997acab 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -1106,6 +1106,10 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa
%45 = call float @llvm.vp.reduce.fmax.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: "llvm.intr.vp.reduce.fmin"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
%46 = call float @llvm.vp.reduce.fmin.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fmaximum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fmaximum = call float @llvm.vp.reduce.fmaximum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
+ ; CHECK: "llvm.intr.vp.reduce.fminimum"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ %fminimum = call float @llvm.vp.reduce.fminimum.v8f32(float %8, <8 x float> %2, <8 x i1> %11, i32 %12)
; CHECK: llvm.select %{{.*}}, %{{.*}}, %{{.*}} : vector<8xi1>, vector<8xi32>
%47 = call <8 x i32> @llvm.vp.select.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12)
; CHECK: "llvm.intr.vp.merge"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi1>, vector<8xi32>, vector<8xi32>, i32) -> vector<8xi32>
@@ -1924,6 +1928,8 @@ declare i32 @llvm.vp.reduce.umin.v8i32(i32, <8 x i32>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index b3776c9a99f97..3839b3dc89e25 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -1085,6 +1085,12 @@ llvm.func @vector_predication_intrinsics(%A: vector<8xi32>, %B: vector<8xi32>,
// CHECK: call float @llvm.vp.reduce.fmin.v8f32
"llvm.intr.vp.reduce.fmin" (%f, %C, %mask, %evl) :
(f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fmaximum.v8f32
+ "llvm.intr.vp.reduce.fmaximum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
+ // CHECK: call float @llvm.vp.reduce.fminimum.v8f32
+ "llvm.intr.vp.reduce.fminimum" (%f, %C, %mask, %evl) :
+ (f32, vector<8xf32>, vector<8xi1>, i32) -> f32
// CHECK: call <8 x i32> @llvm.vp.merge.v8i32
"llvm.intr.vp.merge" (%mask, %A, %B, %evl) :
@@ -2083,6 +2089,8 @@ llvm.func @vector_scmp(%a: vector<4 x i32>, %b: vector<4 x i32>) -> vector<4 x i
// CHECK-DAG: declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
+// CHECK-DAG: declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32)
// CHECK-DAG: declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr captures(none), i32, <8 x i1>, i32)
// CHECK-DAG: declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr captures(none), i32, <8 x i1>, i32)
>From 96000b294da0e6eac904d58292110441da3e39e9 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 12:52:48 +0200
Subject: [PATCH 2/3] Address review: state the neutral requirement instead of
a counterfactual
---
.../CPU/reductions-masked-minmax-f32.mlir | 176 ++++++++++++------
1 file changed, 124 insertions(+), 52 deletions(-)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
index 74ae2da875b91..797d04f786c1f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -3,78 +3,150 @@
// RUN: -shared-libs=%mlir_c_runner_utils | \
// RUN: FileCheck %s
-func.func @entry() {
- %inf = arith.constant 0x7F800000 : f32
- %ninf = arith.constant 0xFF800000 : f32
+func.func @maximumf_finite() {
+ // The neutral (-inf) must order below every active lane. The masked-off lane is the
+ // largest.
+ // max(<-inf>, -5, -7, -5) = -5
%mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
-
- // Ordinary negative data. A neutral value anywhere above the active lanes,
- // such as a negative subnormal, would be returned instead of the maximum.
- %v0 = arith.constant dense<[-5.0, -7.0, -5.0, -9.0]> : vector<4xf32>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
%0 = vector.mask %mask {
- vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
+ vector.print str "maximumf_finite\n"
vector.print %0 : f32
- // CHECK: -5
+ return
+}
+// CHECK-LABEL: maximumf_finite
+// CHECK-NEXT: -5
- // Mirror image for the minimum.
- %v1 = arith.constant dense<[5.0, 7.0, 5.0, 9.0]> : vector<4xf32>
- %1 = vector.mask %mask {
- vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+func.func @minimumf_finite() {
+ // min(<+inf>, 5, 7, 5) = 5
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %1 : f32
- // CHECK: 5
+ vector.print str "minimumf_finite\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_finite
+// CHECK-NEXT: 5
- // Infinite active lanes: the largest finite value would win over them.
- %v2 = vector.broadcast %ninf : f32 to vector<4xf32>
- %2 = vector.mask %mask {
- vector.reduction <maximumf>, %v2 : vector<4xf32> into f32
+func.func @maximumf_inf() {
+ // The neutral must be an infinity: a finite one wins over -inf lanes.
+ // max(<-inf>, -inf, -inf, -inf) = -inf
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %ninf = arith.constant 0xFF800000 : f32
+ %v = vector.broadcast %ninf : f32 to vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %2 : f32
- // CHECK: -inf
+ vector.print str "maximumf_inf\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_inf
+// CHECK-NEXT: -inf
- %v3 = vector.broadcast %inf : f32 to vector<4xf32>
- %3 = vector.mask %mask {
- vector.reduction <minimumf>, %v3 : vector<4xf32> into f32
+func.func @minimumf_inf() {
+ // min(<+inf>, +inf, +inf, +inf) = +inf
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %inf = arith.constant 0x7F800000 : f32
+ %v = vector.broadcast %inf : f32 to vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %3 : f32
- // CHECK: inf
+ vector.print str "minimumf_inf\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_inf
+// CHECK-NEXT: inf
- // No active lane: the result is the identity of the reduction.
- %nomask = arith.constant dense<false> : vector<4xi1>
- %4 = vector.mask %nomask {
- vector.reduction <maximumf>, %v0 : vector<4xf32> into f32
+func.func @maximumf_no_active_lane() {
+ // With no active lane the neutral wins.
+ // max(<-inf>) = -inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %4 : f32
- // CHECK: -inf
+ vector.print str "maximumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_no_active_lane
+// CHECK-NEXT: -inf
- %5 = vector.mask %nomask {
- vector.reduction <minimumf>, %v1 : vector<4xf32> into f32
+func.func @minimumf_no_active_lane() {
+ // min(<+inf>) = +inf
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %5 : f32
- // CHECK: inf
+ vector.print str "minimumf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_no_active_lane
+// CHECK-NEXT: inf
- // An accumulator combines with the masked reduction.
- %acc = arith.constant -3.0 : f32
- %6 = vector.mask %mask {
- vector.reduction <maximumf>, %v0, %acc : vector<4xf32> into f32
+func.func @maximumf_ninf_no_active_lane() {
+ // Under `ninf` the neutral is -FLT_MAX instead of -inf.
+ // max(<-FLT_MAX>) = -FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v fastmath<ninf> : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %6 : f32
- // CHECK: -3
+ vector.print str "maximumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_ninf_no_active_lane
+// CHECK-NEXT: -3.40282e+38
- // Under `ninf` the neutral value is the largest finite value instead. It is
- // still neutral for the finite inputs the flag restricts the reduction to.
- %7 = vector.mask %mask {
- vector.reduction <maximumf>, %v0 fastmath<ninf> : vector<4xf32> into f32
+func.func @minimumf_ninf_no_active_lane() {
+ // min(<+FLT_MAX>) = +FLT_MAX
+ %mask = arith.constant dense<false> : vector<4xi1>
+ %v = arith.constant dense<[5.0, 7.0, 5.0, 1.0]> : vector<4xf32>
+ %0 = vector.mask %mask {
+ vector.reduction <minimumf>, %v fastmath<ninf> : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %7 : f32
- // CHECK: -5
+ vector.print str "minimumf_ninf_no_active_lane\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: minimumf_ninf_no_active_lane
+// CHECK-NEXT: 3.40282e+38
- %8 = vector.mask %mask {
- vector.reduction <minimumf>, %v1 fastmath<ninf> : vector<4xf32> into f32
+func.func @maximumf_acc() {
+ // An accumulator takes the place of the neutral.
+ // max(<-3>, -5, -7, -5) = -3
+ %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
+ %v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
+ %acc = arith.constant -3.0 : f32
+ %0 = vector.mask %mask {
+ vector.reduction <maximumf>, %v, %acc : vector<4xf32> into f32
} : vector<4xi1> -> f32
- vector.print %8 : f32
- // CHECK: 5
+ vector.print str "maximumf_acc\n"
+ vector.print %0 : f32
+ return
+}
+// CHECK-LABEL: maximumf_acc
+// CHECK-NEXT: -3
+func.func @entry() {
+ call @maximumf_finite() : () -> ()
+ call @minimumf_finite() : () -> ()
+ call @maximumf_inf() : () -> ()
+ call @minimumf_inf() : () -> ()
+ call @maximumf_no_active_lane() : () -> ()
+ call @minimumf_no_active_lane() : () -> ()
+ call @maximumf_ninf_no_active_lane() : () -> ()
+ call @minimumf_ninf_no_active_lane() : () -> ()
+ call @maximumf_acc() : () -> ()
return
}
>From dc01e0e7aadf23c7b2b246d4363ea9cdf7cc5df7 Mon Sep 17 00:00:00 2001
From: lorenzo chelini <lchelini at nvidia.com>
Date: Wed, 9 Sep 2026 13:15:44 +0200
Subject: [PATCH 3/3] Remove confusing tests
---
.../CPU/reductions-masked-minmax-f32.mlir | 37 +------------------
1 file changed, 2 insertions(+), 35 deletions(-)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
index 797d04f786c1f..4dcbac06cb205 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/reductions-masked-minmax-f32.mlir
@@ -4,8 +4,8 @@
// RUN: FileCheck %s
func.func @maximumf_finite() {
- // The neutral (-inf) must order below every active lane. The masked-off lane is the
- // largest.
+ // The neutral (-inf) must order below every active lane. The masked-off
+ // lane is the largest.
// max(<-inf>, -5, -7, -5) = -5
%mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
%v = arith.constant dense<[-5.0, -7.0, -5.0, -1.0]> : vector<4xf32>
@@ -33,37 +33,6 @@ func.func @minimumf_finite() {
// CHECK-LABEL: minimumf_finite
// CHECK-NEXT: 5
-func.func @maximumf_inf() {
- // The neutral must be an infinity: a finite one wins over -inf lanes.
- // max(<-inf>, -inf, -inf, -inf) = -inf
- %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
- %ninf = arith.constant 0xFF800000 : f32
- %v = vector.broadcast %ninf : f32 to vector<4xf32>
- %0 = vector.mask %mask {
- vector.reduction <maximumf>, %v : vector<4xf32> into f32
- } : vector<4xi1> -> f32
- vector.print str "maximumf_inf\n"
- vector.print %0 : f32
- return
-}
-// CHECK-LABEL: maximumf_inf
-// CHECK-NEXT: -inf
-
-func.func @minimumf_inf() {
- // min(<+inf>, +inf, +inf, +inf) = +inf
- %mask = arith.constant dense<[true, true, true, false]> : vector<4xi1>
- %inf = arith.constant 0x7F800000 : f32
- %v = vector.broadcast %inf : f32 to vector<4xf32>
- %0 = vector.mask %mask {
- vector.reduction <minimumf>, %v : vector<4xf32> into f32
- } : vector<4xi1> -> f32
- vector.print str "minimumf_inf\n"
- vector.print %0 : f32
- return
-}
-// CHECK-LABEL: minimumf_inf
-// CHECK-NEXT: inf
-
func.func @maximumf_no_active_lane() {
// With no active lane the neutral wins.
// max(<-inf>) = -inf
@@ -141,8 +110,6 @@ func.func @maximumf_acc() {
func.func @entry() {
call @maximumf_finite() : () -> ()
call @minimumf_finite() : () -> ()
- call @maximumf_inf() : () -> ()
- call @minimumf_inf() : () -> ()
call @maximumf_no_active_lane() : () -> ()
call @minimumf_no_active_lane() : () -> ()
call @maximumf_ninf_no_active_lane() : () -> ()
More information about the Mlir-commits
mailing list