[flang-commits] [flang] [flang][OpenACC] Fix reduction init value for minnumf/minimumf/maxnumf/maximumf (PR #187647)
via flang-commits
flang-commits at lists.llvm.org
Fri Mar 20 10:17:30 PDT 2026
https://github.com/khaki3 updated https://github.com/llvm/llvm-project/pull/187647
>From 443b592d92ef300df708878359a1938e0aaa68a8 Mon Sep 17 00:00:00 2001
From: Kazuaki Matsumura <kmatsumura at nvidia.com>
Date: Fri, 20 Mar 2026 00:17:00 -0700
Subject: [PATCH] [flang][OpenACC] Fix reduction init value for
minnumf/minimumf/maxnumf/maximumf
The reduction recipe init region was producing 0.0 instead of the
correct identity value (largest representable float for min, smallest
for max) when the reduction operator was AccMinnumf, AccMinimumf,
AccMaxnumf, or AccMaximumf. Only AccMin and AccMax were handled,
causing the new operator variants to fall through to the else branch
which returns 0.
This caused GPU min reductions to always produce 0.0 since
min(x, 0.0) = 0.0 for all positive x.
Replace the duplicated identity value logic with
arith::getIdentityValue, using a mapping from acc::ReductionOperator
to arith::AtomicRMWKind. Use minimumf/maximumf (which respect
useOnlyFiniteValue) instead of minnumf/maxnumf (whose MLIR identity
is NaN) to get correct finite identity values.
This also fixes a pre-existing bug where the max reduction identity
for floats used getSmallest (smallest subnormal, -1.4e-45) instead
of getLargest with negative (-3.4e+38).
Made-with: Cursor
---
.../OpenACC/Support/FIROpenACCUtils.cpp | 127 +++++++-----------
.../Lower/OpenACC/acc-reduction-maxmin.f90 | 2 +
flang/test/Lower/OpenACC/acc-reduction.f90 | 12 +-
3 files changed, 55 insertions(+), 86 deletions(-)
diff --git a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCUtils.cpp b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCUtils.cpp
index a53ea9216f7ab..3a8f548fefdf6 100644
--- a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCUtils.cpp
+++ b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCUtils.cpp
@@ -271,59 +271,41 @@ std::string fir::acc::getRecipeName(mlir::acc::RecipeKind kind, Type type,
return ::getRecipeName(kind, type, kindMap, bounds, reductionOp);
}
-/// Get the initial value for reduction operator.
-template <typename R>
-static R getReductionInitValue(mlir::acc::ReductionOperator op, mlir::Type ty) {
- if (op == mlir::acc::ReductionOperator::AccMin) {
- // min init value -> largest
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt::getSignedMaxValue(ty.getIntOrFloatBitWidth());
- }
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
- assert(floatTy && "expect float type");
- return llvm::APFloat::getLargest(floatTy.getFloatSemantics(),
- /*negative=*/false);
- }
- } else if (op == mlir::acc::ReductionOperator::AccMax) {
- // max init value -> smallest
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt::getSignedMinValue(ty.getIntOrFloatBitWidth());
- }
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
- assert(floatTy && "expect float type");
- return llvm::APFloat::getSmallest(floatTy.getFloatSemantics(),
- /*negative=*/true);
- }
- } else if (op == mlir::acc::ReductionOperator::AccIand) {
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer type");
- unsigned bits = ty.getIntOrFloatBitWidth();
- return llvm::APInt::getAllOnes(bits);
- }
- } else {
- assert(op != mlir::acc::ReductionOperator::AccNone);
- // +, ior, ieor init value -> 0
- // * init value -> 1
- int64_t value = (op == mlir::acc::ReductionOperator::AccMul) ? 1 : 0;
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt(ty.getIntOrFloatBitWidth(), value, true);
- }
-
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- assert(mlir::isa<mlir::FloatType>(ty) && "expect float type");
- auto floatTy = mlir::dyn_cast<mlir::FloatType>(ty);
- return llvm::APFloat(floatTy.getFloatSemantics(), value);
- }
-
- if constexpr (std::is_same_v<R, int64_t>)
- return value;
+/// Map acc::ReductionOperator to arith::AtomicRMWKind for identity value
+/// computation. Uses minimumf/maximumf instead of minnumf/maxnumf because
+/// arith::getIdentityValueAttr for minnumf/maxnumf returns NaN (the IEEE 754
+/// identity), which doesn't work with comparison-based reductions on GPU.
+/// minimumf/maximumf identity with useOnlyFiniteValue gives the correct
+/// finite extreme value (FLT_MAX / -FLT_MAX).
+static mlir::arith::AtomicRMWKind
+getAtomicRMWKindForIdentity(mlir::acc::ReductionOperator op, mlir::Type ty) {
+ bool isFloat = mlir::isa<mlir::FloatType>(ty);
+ switch (op) {
+ case mlir::acc::ReductionOperator::AccAdd:
+ return isFloat ? mlir::arith::AtomicRMWKind::addf
+ : mlir::arith::AtomicRMWKind::addi;
+ case mlir::acc::ReductionOperator::AccMul:
+ return isFloat ? mlir::arith::AtomicRMWKind::mulf
+ : mlir::arith::AtomicRMWKind::muli;
+ case mlir::acc::ReductionOperator::AccMin:
+ case mlir::acc::ReductionOperator::AccMinnumf:
+ case mlir::acc::ReductionOperator::AccMinimumf:
+ return isFloat ? mlir::arith::AtomicRMWKind::minimumf
+ : mlir::arith::AtomicRMWKind::mins;
+ case mlir::acc::ReductionOperator::AccMax:
+ case mlir::acc::ReductionOperator::AccMaxnumf:
+ case mlir::acc::ReductionOperator::AccMaximumf:
+ return isFloat ? mlir::arith::AtomicRMWKind::maximumf
+ : mlir::arith::AtomicRMWKind::maxs;
+ case mlir::acc::ReductionOperator::AccIand:
+ return mlir::arith::AtomicRMWKind::andi;
+ case mlir::acc::ReductionOperator::AccIor:
+ return mlir::arith::AtomicRMWKind::ori;
+ case mlir::acc::ReductionOperator::AccXor:
+ return mlir::arith::AtomicRMWKind::xori;
+ default:
+ llvm_unreachable("unsupported acc::ReductionOperator");
}
- llvm_unreachable("OpenACC reduction unsupported type");
}
/// Return a constant with the initial value for the reduction operator and
@@ -337,39 +319,24 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
op == mlir::acc::ReductionOperator::AccEqv ||
op == mlir::acc::ReductionOperator::AccNeqv) {
assert(mlir::isa<fir::LogicalType>(ty) && "expect fir.logical type");
- bool value = true; // .true. for .and. and .eqv.
- if (op == mlir::acc::ReductionOperator::AccLor ||
- op == mlir::acc::ReductionOperator::AccNeqv)
- value = false; // .false. for .or. and .neqv.
+ bool value = (op == mlir::acc::ReductionOperator::AccLand ||
+ op == mlir::acc::ReductionOperator::AccEqv);
return builder.createBool(loc, value);
}
- if (ty.isIntOrIndex())
- return mlir::arith::ConstantOp::create(
- builder, loc, ty,
- builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
- if (op == mlir::acc::ReductionOperator::AccMin ||
- op == mlir::acc::ReductionOperator::AccMax) {
- if (mlir::isa<mlir::ComplexType>(ty))
- llvm::report_fatal_error(
- "min/max reduction not supported for complex type");
- if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
- return mlir::arith::ConstantOp::create(
- builder, loc, ty,
- builder.getFloatAttr(ty,
- getReductionInitValue<llvm::APFloat>(op, ty)));
- } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
- return mlir::arith::ConstantOp::create(
- builder, loc, ty,
- builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
- } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
- mlir::Type floatTy = cmplxTy.getElementType();
- mlir::Value realInit = builder.createRealConstant(
- loc, floatTy, getReductionInitValue<int64_t>(op, cmplxTy));
- mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0);
+ if (auto cmplxTy = mlir::dyn_cast<mlir::ComplexType>(ty)) {
+ mlir::arith::AtomicRMWKind kind =
+ getAtomicRMWKindForIdentity(op, cmplxTy.getElementType());
+ mlir::Value realInit = mlir::arith::getIdentityValue(
+ kind, cmplxTy.getElementType(), builder, loc,
+ /*useOnlyFiniteValue=*/true);
+ mlir::Value imagInit =
+ builder.createRealConstant(loc, cmplxTy.getElementType(), 0.0);
return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit,
imagInit);
}
- llvm::report_fatal_error("Unsupported OpenACC reduction type");
+ mlir::arith::AtomicRMWKind kind = getAtomicRMWKindForIdentity(op, ty);
+ return mlir::arith::getIdentityValue(kind, ty, builder, loc,
+ /*useOnlyFiniteValue=*/true);
}
static llvm::SmallVector<mlir::Value>
diff --git a/flang/test/Lower/OpenACC/acc-reduction-maxmin.f90 b/flang/test/Lower/OpenACC/acc-reduction-maxmin.f90
index 2d0746decf29d..cd6e3fd2fb07b 100644
--- a/flang/test/Lower/OpenACC/acc-reduction-maxmin.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction-maxmin.f90
@@ -61,6 +61,7 @@ end subroutine acc_array_reduction_min
! EXTREMUM: %[[MINIMUMF_0:.*]] = arith.minimumf %{{.*}}, %{{.*}} fastmath<contract> : f32
! EXTREMUM-LABEL: acc.reduction.recipe @reduction_minimumf_ref_f32 : !fir.ref<f32> reduction_operator <minimumf> init {
+! EXTREMUM: %[[CST:.*]] = arith.constant 3.40282347E+38 : f32
! EXTREMUM: } combiner {
! EXTREMUM: %[[MINIMUMF_0:.*]] = arith.minimumf %{{.*}}, %{{.*}} fastmath<contract> : f32
@@ -79,6 +80,7 @@ end subroutine acc_array_reduction_min
! EXTREMENUM: %[[MINNUMF_0:.*]] = arith.minnumf %{{.*}}, %{{.*}} fastmath<contract> : f32
! EXTREMENUM-LABEL: acc.reduction.recipe @reduction_minnumf_ref_f32 : !fir.ref<f32> reduction_operator <minnumf> init {
+! EXTREMENUM: %[[CST:.*]] = arith.constant 3.40282347E+38 : f32
! EXTREMENUM: } combiner {
! EXTREMENUM: %[[MINNUMF_0:.*]] = arith.minnumf %{{.*}}, %{{.*}} fastmath<contract> : f32
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index b17cea99ae44a..49789efc545ba 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -40,7 +40,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_UxUxf32 : !fir.box<!fir.array<?x?xf32>> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.box<!fir.array<?x?xf32>>):
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: %[[CONSTANT_1:.*]] = arith.constant 0 : index
! CHECK: %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[VAL_0]], %[[CONSTANT_1]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
! CHECK: %[[CONSTANT_2:.*]] = arith.constant 1 : index
@@ -127,7 +127,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_box_ptr_Uxf32 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: %[[LOAD_0:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
! CHECK: %[[CONSTANT_1:.*]] = arith.constant 0 : index
! CHECK: %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[LOAD_0]], %[[CONSTANT_1]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
@@ -196,7 +196,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_box_heap_Uxf32 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>):
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: %[[LOAD_0:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
! CHECK: %[[CONSTANT_1:.*]] = arith.constant 0 : index
! CHECK: %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[LOAD_0]], %[[CONSTANT_1]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
@@ -333,7 +333,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_Uxf32 : !fir.box<!fir.array<?xf32>> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>):
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: %[[CONSTANT_1:.*]] = arith.constant 0 : index
! CHECK: %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[VAL_0]], %[[CONSTANT_1]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
! CHECK: %[[SHAPE_0:.*]] = fir.shape %[[BOX_DIMS_0]]#1 : (index) -> !fir.shape<1>
@@ -826,7 +826,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.array<100xf32>>):
! CHECK: %[[ALLOCA_0:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "acc.reduction.init"}
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: %[[CONSTANT_1:.*]] = arith.constant 100 : index
! CHECK: %[[SHAPE_0:.*]] = fir.shape %[[CONSTANT_1]] : (index) -> !fir.shape<1>
! CHECK: %[[SHAPE_1:.*]] = fir.shape %[[CONSTANT_1]] : (index) -> !fir.shape<1>
@@ -861,7 +861,7 @@
! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_f32 : !fir.ref<f32> reduction_operator <max> init {
! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref<f32>):
! CHECK: %[[ALLOCA_0:.*]] = fir.alloca f32 {bindc_name = "acc.reduction.init"}
-! CHECK: %[[CONSTANT_0:.*]] = arith.constant -1.401300e-45 : f32
+! CHECK: %[[CONSTANT_0:.*]] = arith.constant -3.40282347E+38 : f32
! CHECK: hlfir.assign %[[CONSTANT_0]] to %[[ALLOCA_0]] temporary_lhs : f32, !fir.ref<f32>
! CHECK: acc.yield %[[ALLOCA_0]] : !fir.ref<f32>
More information about the flang-commits
mailing list