[flang-commits] [flang] [mlir] [OpenACC] unify reduction and private-like init region recipes (PR #140652)
Scott Manley via flang-commits
flang-commits at lists.llvm.org
Mon May 19 17:33:47 PDT 2025
https://github.com/rscottmanley created https://github.com/llvm/llvm-project/pull/140652
Between firstprivate, private and reduction init regions, the difference is largely whether or not the temp that is created is initialized or not. Some recent fixes were made to privatization (#135698, #137869) but did not get propagated to reductions, even though they need to return the yield the same things from their init regions.
To mitigate this discrepancy in the future, refactor the init region recipes so they can be shared between the three recipe ops.
Also add "none" to the OpenACC_ReductionOperator enum for better error checking.
>From eed6fd1295c8613f8d3ab76262e8f758f6fac462 Mon Sep 17 00:00:00 2001
From: Scott Manley <scmanley at nvidia.com>
Date: Mon, 19 May 2025 16:40:10 -0700
Subject: [PATCH] [OpenACC] unify reduction and private-like init region
recipes
Between firstprivate, private and reduction init regions, the
difference is largely whether or not the temp that is created is
initialized or not. Some recent fixes were made to privatization
(#135698, #137869) but did not get propagated to reductions, even
though they essentially do the same thing.
To mitigate this descrepency in the future, refactor the init
region recipes so they can be shared between the three recipe ops.
Also add "none" to the OpenACC_ReductionOperator enum for better
error checking.
---
flang/include/flang/Lower/OpenACC.h | 4 +-
flang/lib/Lower/OpenACC.cpp | 458 ++++++++----------
flang/test/Lower/OpenACC/acc-reduction.f90 | 16 +-
.../mlir/Dialect/OpenACC/OpenACCOps.td | 27 +-
4 files changed, 233 insertions(+), 272 deletions(-)
diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h
index bbe3b01fdb29d..dad841863ac00 100644
--- a/flang/include/flang/Lower/OpenACC.h
+++ b/flang/include/flang/Lower/OpenACC.h
@@ -85,7 +85,7 @@ void genOpenACCRoutineConstruct(
/// Get a acc.private.recipe op for the given type or create it if it does not
/// exist yet.
-mlir::acc::PrivateRecipeOp createOrGetPrivateRecipe(mlir::OpBuilder &,
+mlir::acc::PrivateRecipeOp createOrGetPrivateRecipe(fir::FirOpBuilder &,
llvm::StringRef,
mlir::Location, mlir::Type);
@@ -99,7 +99,7 @@ createOrGetReductionRecipe(fir::FirOpBuilder &, llvm::StringRef, mlir::Location,
/// Get a acc.firstprivate.recipe op for the given type or create it if it does
/// not exist yet.
mlir::acc::FirstprivateRecipeOp
-createOrGetFirstprivateRecipe(mlir::OpBuilder &, llvm::StringRef,
+createOrGetFirstprivateRecipe(fir::FirOpBuilder &, llvm::StringRef,
mlir::Location, mlir::Type,
llvm::SmallVector<mlir::Value> &);
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index e1918288d6de3..98f6adf8f17c6 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -843,22 +843,147 @@ fir::ShapeOp genShapeOp(mlir::OpBuilder &builder, fir::SequenceType seqTy,
return builder.create<fir::ShapeOp>(loc, extents);
}
+/// Get the initial value for reduction operator.
+template <typename R>
+static R getReductionInitValue(mlir::acc::ReductionOperator op, mlir::Type ty) {
+ if (op == mlir::acc::ReductionOperator::AccMin) {
+ // min init value -> largest
+ if constexpr (std::is_same_v<R, llvm::APInt>) {
+ assert(ty.isIntOrIndex() && "expect integer or index type");
+ return llvm::APInt::getSignedMaxValue(ty.getIntOrFloatBitWidth());
+ }
+ if constexpr (std::is_same_v<R, llvm::APFloat>) {
+ auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
+ assert(floatTy && "expect float type");
+ return llvm::APFloat::getLargest(floatTy.getFloatSemantics(),
+ /*negative=*/false);
+ }
+ } else if (op == mlir::acc::ReductionOperator::AccMax) {
+ // max init value -> smallest
+ if constexpr (std::is_same_v<R, llvm::APInt>) {
+ assert(ty.isIntOrIndex() && "expect integer or index type");
+ return llvm::APInt::getSignedMinValue(ty.getIntOrFloatBitWidth());
+ }
+ if constexpr (std::is_same_v<R, llvm::APFloat>) {
+ auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
+ assert(floatTy && "expect float type");
+ return llvm::APFloat::getSmallest(floatTy.getFloatSemantics(),
+ /*negative=*/true);
+ }
+ } else if (op == mlir::acc::ReductionOperator::AccIand) {
+ if constexpr (std::is_same_v<R, llvm::APInt>) {
+ assert(ty.isIntOrIndex() && "expect integer type");
+ unsigned bits = ty.getIntOrFloatBitWidth();
+ return llvm::APInt::getAllOnes(bits);
+ }
+ } else {
+ assert(op != mlir::acc::ReductionOperator::AccNone);
+ // +, ior, ieor init value -> 0
+ // * init value -> 1
+ int64_t value = (op == mlir::acc::ReductionOperator::AccMul) ? 1 : 0;
+ if constexpr (std::is_same_v<R, llvm::APInt>) {
+ assert(ty.isIntOrIndex() && "expect integer or index type");
+ return llvm::APInt(ty.getIntOrFloatBitWidth(), value, true);
+ }
+
+ if constexpr (std::is_same_v<R, llvm::APFloat>) {
+ assert(mlir::isa<mlir::FloatType>(ty) && "expect float type");
+ auto floatTy = mlir::dyn_cast<mlir::FloatType>(ty);
+ return llvm::APFloat(floatTy.getFloatSemantics(), value);
+ }
+
+ if constexpr (std::is_same_v<R, int64_t>)
+ return value;
+ }
+ llvm_unreachable("OpenACC reduction unsupported type");
+}
+
+/// Return a constant with the initial value for the reduction operator and
+/// type combination.
+static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
+ mlir::Location loc, mlir::Type ty,
+ mlir::acc::ReductionOperator op) {
+ if (op == mlir::acc::ReductionOperator::AccLand ||
+ op == mlir::acc::ReductionOperator::AccLor ||
+ op == mlir::acc::ReductionOperator::AccEqv ||
+ op == mlir::acc::ReductionOperator::AccNeqv) {
+ assert(mlir::isa<fir::LogicalType>(ty) && "expect fir.logical type");
+ bool value = true; // .true. for .and. and .eqv.
+ if (op == mlir::acc::ReductionOperator::AccLor ||
+ op == mlir::acc::ReductionOperator::AccNeqv)
+ value = false; // .false. for .or. and .neqv.
+ return builder.createBool(loc, value);
+ }
+ if (ty.isIntOrIndex())
+ return builder.create<mlir::arith::ConstantOp>(
+ loc, ty,
+ builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
+ if (op == mlir::acc::ReductionOperator::AccMin ||
+ op == mlir::acc::ReductionOperator::AccMax) {
+ if (mlir::isa<mlir::ComplexType>(ty))
+ llvm::report_fatal_error(
+ "min/max reduction not supported for complex type");
+ if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
+ return builder.create<mlir::arith::ConstantOp>(
+ loc, ty,
+ builder.getFloatAttr(ty,
+ getReductionInitValue<llvm::APFloat>(op, ty)));
+ } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
+ return builder.create<mlir::arith::ConstantOp>(
+ loc, ty,
+ builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
+ } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
+ mlir::Type floatTy = cmplxTy.getElementType();
+ mlir::Value realInit = builder.createRealConstant(
+ loc, floatTy, getReductionInitValue<int64_t>(op, cmplxTy));
+ mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0);
+ return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit,
+ imagInit);
+ }
+
+ if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty))
+ return getReductionInitValue(builder, loc, seqTy.getEleTy(), op);
+
+ if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty))
+ return getReductionInitValue(builder, loc, boxTy.getEleTy(), op);
+
+ if (auto heapTy = mlir::dyn_cast<fir::HeapType>(ty))
+ return getReductionInitValue(builder, loc, heapTy.getEleTy(), op);
+
+ if (auto ptrTy = mlir::dyn_cast<fir::PointerType>(ty))
+ return getReductionInitValue(builder, loc, ptrTy.getEleTy(), op);
+
+ llvm::report_fatal_error("Unsupported OpenACC reduction type");
+}
+
template <typename RecipeOp>
-static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
- mlir::Type argTy, mlir::Location loc) {
+static void genPrivateLikeInitRegion(fir::FirOpBuilder &builder,
+ RecipeOp recipe, mlir::Type argTy,
+ mlir::Location loc,
+ mlir::Value initValue) {
mlir::Value retVal = recipe.getInitRegion().front().getArgument(0);
mlir::Type unwrappedTy = fir::unwrapRefType(argTy);
+ llvm::StringRef initName;
+ if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>)
+ initName = accReductionInitName;
+ else
+ initName = accPrivateInitName;
+
auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp {
auto alloca = builder.create<fir::AllocaOp>(loc, ty);
return builder.create<hlfir::DeclareOp>(
- loc, alloca, accPrivateInitName, /*shape=*/nullptr,
- llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
- fir::FortranVariableFlagsAttr{});
+ loc, alloca, initName, /*shape=*/nullptr, llvm::ArrayRef<mlir::Value>{},
+ /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
};
if (fir::isa_trivial(unwrappedTy)) {
- retVal = getDeclareOpForType(unwrappedTy).getBase();
+ auto declareOp = getDeclareOpForType(unwrappedTy);
+ if (initValue) {
+ auto convert = builder.createConvert(loc, unwrappedTy, initValue);
+ builder.create<fir::StoreOp>(loc, convert, declareOp.getBase());
+ }
+ retVal = declareOp.getBase();
} else if (auto seqTy =
mlir::dyn_cast_or_null<fir::SequenceType>(unwrappedTy)) {
if (fir::isa_trivial(seqTy.getEleTy())) {
@@ -877,8 +1002,34 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
auto alloca = builder.create<fir::AllocaOp>(
loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
auto declareOp = builder.create<hlfir::DeclareOp>(
- loc, alloca, accPrivateInitName, shape, llvm::ArrayRef<mlir::Value>{},
+ loc, alloca, initName, shape, llvm::ArrayRef<mlir::Value>{},
/*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
+
+ if (initValue) {
+ mlir::Type idxTy = builder.getIndexType();
+ mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
+ llvm::SmallVector<fir::DoLoopOp> loops;
+ llvm::SmallVector<mlir::Value> ivs;
+
+ if (seqTy.hasDynamicExtents()) {
+ builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
+ } else {
+ for (auto ext : seqTy.getShape()) {
+ auto lb = builder.createIntegerConstant(loc, idxTy, 0);
+ auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
+ auto step = builder.createIntegerConstant(loc, idxTy, 1);
+ auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
+ /*unordered=*/false);
+ builder.setInsertionPointToStart(loop.getBody());
+ loops.push_back(loop);
+ ivs.push_back(loop.getInductionVar());
+ }
+ auto coord = builder.create<fir::CoordinateOp>(
+ loc, refTy, declareOp.getBase(), ivs);
+ builder.create<fir::StoreOp>(loc, initValue, coord);
+ builder.setInsertionPointAfter(loops[0]);
+ }
+ }
retVal = declareOp.getBase();
}
} else if (auto boxTy =
@@ -909,25 +1060,29 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
retVal = temp;
}
} else {
- TODO(loc, "Unsupported boxed type in OpenACC privatization");
+ TODO(loc, "Unsupported boxed type for OpenACC private-like recipe");
+ }
+ if (initValue) {
+ builder.create<hlfir::AssignOp>(loc, initValue, retVal);
}
}
builder.create<mlir::acc::YieldOp>(loc, retVal);
}
-mlir::acc::PrivateRecipeOp
-Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder,
- llvm::StringRef recipeName,
- mlir::Location loc, mlir::Type ty) {
- mlir::ModuleOp mod =
- builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
- if (auto recipe = mod.lookupSymbol<mlir::acc::PrivateRecipeOp>(recipeName))
- return recipe;
-
- auto crtPos = builder.saveInsertionPoint();
+template <typename RecipeOp>
+static RecipeOp genRecipeOp(
+ fir::FirOpBuilder &builder, mlir::ModuleOp mod, llvm::StringRef recipeName,
+ mlir::Location loc, mlir::Type ty,
+ mlir::acc::ReductionOperator op = mlir::acc::ReductionOperator::AccNone) {
mlir::OpBuilder modBuilder(mod.getBodyRegion());
- auto recipe =
- modBuilder.create<mlir::acc::PrivateRecipeOp>(loc, recipeName, ty);
+ RecipeOp recipe;
+ if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
+ recipe = modBuilder.create<mlir::acc::ReductionRecipeOp>(loc, recipeName,
+ ty, op);
+ } else {
+ recipe = modBuilder.create<RecipeOp>(loc, recipeName, ty);
+ }
+
llvm::SmallVector<mlir::Type> argsTy{ty};
llvm::SmallVector<mlir::Location> argsLoc{loc};
if (auto refTy = mlir::dyn_cast_or_null<fir::ReferenceType>(ty)) {
@@ -945,9 +1100,28 @@ Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder,
builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
argsTy, argsLoc);
builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
- genPrivateLikeInitRegion<mlir::acc::PrivateRecipeOp>(builder, recipe, ty,
- loc);
- builder.restoreInsertionPoint(crtPos);
+ mlir::Value initValue;
+ if constexpr (std::is_same_v<RecipeOp, mlir::acc::ReductionRecipeOp>) {
+ assert(op != mlir::acc::ReductionOperator::AccNone);
+ initValue = getReductionInitValue(builder, loc, fir::unwrapRefType(ty), op);
+ }
+ genPrivateLikeInitRegion<RecipeOp>(builder, recipe, ty, loc, initValue);
+ return recipe;
+}
+
+mlir::acc::PrivateRecipeOp
+Fortran::lower::createOrGetPrivateRecipe(fir::FirOpBuilder &builder,
+ llvm::StringRef recipeName,
+ mlir::Location loc, mlir::Type ty) {
+ mlir::ModuleOp mod =
+ builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
+ if (auto recipe = mod.lookupSymbol<mlir::acc::PrivateRecipeOp>(recipeName))
+ return recipe;
+
+ auto ip = builder.saveInsertionPoint();
+ auto recipe = genRecipeOp<mlir::acc::PrivateRecipeOp>(builder, mod,
+ recipeName, loc, ty);
+ builder.restoreInsertionPoint(ip);
return recipe;
}
@@ -1064,7 +1238,7 @@ static hlfir::Entity genDesignateWithTriplets(
}
mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
- mlir::OpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc,
+ fir::FirOpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc,
mlir::Type ty, llvm::SmallVector<mlir::Value> &bounds) {
mlir::ModuleOp mod =
builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
@@ -1072,28 +1246,9 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
mod.lookupSymbol<mlir::acc::FirstprivateRecipeOp>(recipeName))
return recipe;
- auto crtPos = builder.saveInsertionPoint();
- mlir::OpBuilder modBuilder(mod.getBodyRegion());
- auto recipe =
- modBuilder.create<mlir::acc::FirstprivateRecipeOp>(loc, recipeName, ty);
- llvm::SmallVector<mlir::Type> initArgsTy{ty};
- llvm::SmallVector<mlir::Location> initArgsLoc{loc};
- auto refTy = fir::unwrapRefType(ty);
- if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(refTy)) {
- if (seqTy.hasDynamicExtents()) {
- mlir::Type idxTy = builder.getIndexType();
- for (unsigned i = 0; i < seqTy.getDimension(); ++i) {
- initArgsTy.push_back(idxTy);
- initArgsLoc.push_back(loc);
- }
- }
- }
- builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
- initArgsTy, initArgsLoc);
- builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
- genPrivateLikeInitRegion<mlir::acc::FirstprivateRecipeOp>(builder, recipe, ty,
- loc);
-
+ auto ip = builder.saveInsertionPoint();
+ auto recipe = genRecipeOp<mlir::acc::FirstprivateRecipeOp>(builder, mod,
+ recipeName, loc, ty);
bool allConstantBound = areAllBoundConstant(bounds);
llvm::SmallVector<mlir::Type> argsTy{ty, ty};
llvm::SmallVector<mlir::Location> argsLoc{loc, loc};
@@ -1167,7 +1322,7 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
}
builder.create<mlir::acc::TerminatorOp>(loc);
- builder.restoreInsertionPoint(crtPos);
+ builder.restoreInsertionPoint(ip);
return recipe;
}
@@ -1326,188 +1481,6 @@ getReductionOperator(const Fortran::parser::ReductionOperator &op) {
llvm_unreachable("unexpected reduction operator");
}
-/// Get the initial value for reduction operator.
-template <typename R>
-static R getReductionInitValue(mlir::acc::ReductionOperator op, mlir::Type ty) {
- if (op == mlir::acc::ReductionOperator::AccMin) {
- // min init value -> largest
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt::getSignedMaxValue(ty.getIntOrFloatBitWidth());
- }
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
- assert(floatTy && "expect float type");
- return llvm::APFloat::getLargest(floatTy.getFloatSemantics(),
- /*negative=*/false);
- }
- } else if (op == mlir::acc::ReductionOperator::AccMax) {
- // max init value -> smallest
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt::getSignedMinValue(ty.getIntOrFloatBitWidth());
- }
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty);
- assert(floatTy && "expect float type");
- return llvm::APFloat::getSmallest(floatTy.getFloatSemantics(),
- /*negative=*/true);
- }
- } else if (op == mlir::acc::ReductionOperator::AccIand) {
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer type");
- unsigned bits = ty.getIntOrFloatBitWidth();
- return llvm::APInt::getAllOnes(bits);
- }
- } else {
- // +, ior, ieor init value -> 0
- // * init value -> 1
- int64_t value = (op == mlir::acc::ReductionOperator::AccMul) ? 1 : 0;
- if constexpr (std::is_same_v<R, llvm::APInt>) {
- assert(ty.isIntOrIndex() && "expect integer or index type");
- return llvm::APInt(ty.getIntOrFloatBitWidth(), value, true);
- }
-
- if constexpr (std::is_same_v<R, llvm::APFloat>) {
- assert(mlir::isa<mlir::FloatType>(ty) && "expect float type");
- auto floatTy = mlir::dyn_cast<mlir::FloatType>(ty);
- return llvm::APFloat(floatTy.getFloatSemantics(), value);
- }
-
- if constexpr (std::is_same_v<R, int64_t>)
- return value;
- }
- llvm_unreachable("OpenACC reduction unsupported type");
-}
-
-/// Return a constant with the initial value for the reduction operator and
-/// type combination.
-static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
- mlir::Location loc, mlir::Type ty,
- mlir::acc::ReductionOperator op) {
- if (op == mlir::acc::ReductionOperator::AccLand ||
- op == mlir::acc::ReductionOperator::AccLor ||
- op == mlir::acc::ReductionOperator::AccEqv ||
- op == mlir::acc::ReductionOperator::AccNeqv) {
- assert(mlir::isa<fir::LogicalType>(ty) && "expect fir.logical type");
- bool value = true; // .true. for .and. and .eqv.
- if (op == mlir::acc::ReductionOperator::AccLor ||
- op == mlir::acc::ReductionOperator::AccNeqv)
- value = false; // .false. for .or. and .neqv.
- return builder.createBool(loc, value);
- }
- if (ty.isIntOrIndex())
- return builder.create<mlir::arith::ConstantOp>(
- loc, ty,
- builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
- if (op == mlir::acc::ReductionOperator::AccMin ||
- op == mlir::acc::ReductionOperator::AccMax) {
- if (mlir::isa<mlir::ComplexType>(ty))
- llvm::report_fatal_error(
- "min/max reduction not supported for complex type");
- if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
- return builder.create<mlir::arith::ConstantOp>(
- loc, ty,
- builder.getFloatAttr(ty,
- getReductionInitValue<llvm::APFloat>(op, ty)));
- } else if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty)) {
- return builder.create<mlir::arith::ConstantOp>(
- loc, ty,
- builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
- } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
- mlir::Type floatTy = cmplxTy.getElementType();
- mlir::Value realInit = builder.createRealConstant(
- loc, floatTy, getReductionInitValue<int64_t>(op, cmplxTy));
- mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0);
- return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit,
- imagInit);
- }
-
- if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty))
- return getReductionInitValue(builder, loc, seqTy.getEleTy(), op);
-
- if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty))
- return getReductionInitValue(builder, loc, boxTy.getEleTy(), op);
-
- if (auto heapTy = mlir::dyn_cast<fir::HeapType>(ty))
- return getReductionInitValue(builder, loc, heapTy.getEleTy(), op);
-
- if (auto ptrTy = mlir::dyn_cast<fir::PointerType>(ty))
- return getReductionInitValue(builder, loc, ptrTy.getEleTy(), op);
-
- llvm::report_fatal_error("Unsupported OpenACC reduction type");
-}
-
-static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
- mlir::Location loc, mlir::Type ty,
- mlir::acc::ReductionOperator op) {
- ty = fir::unwrapRefType(ty);
- mlir::Value initValue = getReductionInitValue(builder, loc, ty, op);
- if (fir::isa_trivial(ty)) {
- mlir::Value alloca = builder.create<fir::AllocaOp>(loc, ty);
- auto declareOp = builder.create<hlfir::DeclareOp>(
- loc, alloca, accReductionInitName, /*shape=*/nullptr,
- llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
- fir::FortranVariableFlagsAttr{});
- builder.create<fir::StoreOp>(loc, builder.createConvert(loc, ty, initValue),
- declareOp.getBase());
- return declareOp.getBase();
- } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
- if (fir::isa_trivial(seqTy.getEleTy())) {
- mlir::Value shape;
- auto extents = builder.getBlock()->getArguments().drop_front(1);
- if (seqTy.hasDynamicExtents())
- shape = builder.create<fir::ShapeOp>(loc, extents);
- else
- shape = genShapeOp(builder, seqTy, loc);
- mlir::Value alloca = builder.create<fir::AllocaOp>(
- loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
- auto declareOp = builder.create<hlfir::DeclareOp>(
- loc, alloca, accReductionInitName, shape,
- llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
- fir::FortranVariableFlagsAttr{});
- mlir::Type idxTy = builder.getIndexType();
- mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
- llvm::SmallVector<fir::DoLoopOp> loops;
- llvm::SmallVector<mlir::Value> ivs;
-
- if (seqTy.hasDynamicExtents()) {
- builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
- return declareOp.getBase();
- }
- for (auto ext : seqTy.getShape()) {
- auto lb = builder.createIntegerConstant(loc, idxTy, 0);
- auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
- auto step = builder.createIntegerConstant(loc, idxTy, 1);
- auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
- /*unordered=*/false);
- builder.setInsertionPointToStart(loop.getBody());
- loops.push_back(loop);
- ivs.push_back(loop.getInductionVar());
- }
- auto coord = builder.create<fir::CoordinateOp>(loc, refTy,
- declareOp.getBase(), ivs);
- builder.create<fir::StoreOp>(loc, initValue, coord);
- builder.setInsertionPointAfter(loops[0]);
- return declareOp.getBase();
- }
- } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
- mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
- if (!fir::isa_trivial(innerTy) && !mlir::isa<fir::SequenceType>(innerTy))
- TODO(loc, "Unsupported boxed type for reduction");
- // Create the private copy from the initial fir.box.
- hlfir::Entity source = hlfir::Entity{builder.getBlock()->getArgument(0)};
- auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
- mlir::Value newBox = temp;
- if (!mlir::isa<fir::BaseBoxType>(temp.getType())) {
- newBox = builder.create<fir::EmboxOp>(loc, boxTy, temp);
- }
- builder.create<hlfir::AssignOp>(loc, initValue, newBox);
- return newBox;
- }
- llvm::report_fatal_error("Unsupported OpenACC reduction type");
-}
-
template <typename Op>
static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder,
mlir::Location loc, mlir::Value value1,
@@ -1799,27 +1772,10 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe(
if (auto recipe = mod.lookupSymbol<mlir::acc::ReductionRecipeOp>(recipeName))
return recipe;
- auto crtPos = builder.saveInsertionPoint();
- mlir::OpBuilder modBuilder(mod.getBodyRegion());
- auto recipe =
- modBuilder.create<mlir::acc::ReductionRecipeOp>(loc, recipeName, ty, op);
- llvm::SmallVector<mlir::Type> initArgsTy{ty};
- llvm::SmallVector<mlir::Location> initArgsLoc{loc};
- mlir::Type refTy = fir::unwrapRefType(ty);
- if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(refTy)) {
- if (seqTy.hasDynamicExtents()) {
- mlir::Type idxTy = builder.getIndexType();
- for (unsigned i = 0; i < seqTy.getDimension(); ++i) {
- initArgsTy.push_back(idxTy);
- initArgsLoc.push_back(loc);
- }
- }
- }
- builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
- initArgsTy, initArgsLoc);
- builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
- mlir::Value initValue = genReductionInitRegion(builder, loc, ty, op);
- builder.create<mlir::acc::YieldOp>(loc, initValue);
+ auto ip = builder.saveInsertionPoint();
+
+ auto recipe = genRecipeOp<mlir::acc::ReductionRecipeOp>(
+ builder, mod, recipeName, loc, ty, op);
// The two first block arguments are the two values to be combined.
// The next arguments are the iteration ranges (lb, ub, step) to be used
@@ -1846,7 +1802,7 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe(
mlir::Value v2 = recipe.getCombinerRegion().front().getArgument(1);
genCombiner(builder, loc, op, ty, v1, v2, recipe, bounds, allConstantBound);
builder.create<mlir::acc::YieldOp>(loc, v1);
- builder.restoreInsertionPoint(crtPos);
+ builder.restoreInsertionPoint(ip);
return recipe;
}
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 0d97c298f8d24..20b5ad28f78a1 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -39,9 +39,11 @@
! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
-! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
-! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
+! CHECK: %[[STORAGE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK: %[[BOXTEMP:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>>
+! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[BOXTEMP]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
! CHECK: } combiner {
! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
! CHECK: %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
@@ -74,9 +76,11 @@
! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1>
! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array<?xf32>, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""}
-! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
-! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box<!fir.array<?xf32>>
-! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box<!fir.array<?xf32>>
+! CHECK: %[[STORAGE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>)
+! CHECK: %[[BOXTEMP:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[BOXTEMP]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
! CHECK: } combiner {
! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>):
! CHECK: %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 3c22aeb9a1ff7..b9148dc088a6a 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -35,22 +35,23 @@ class OpenACC_Op<string mnemonic, list<Trait> traits = []> :
Op<OpenACC_Dialect, mnemonic, traits>;
// Reduction operation enumeration.
-def OpenACC_ReductionOperatorAdd : I32EnumAttrCase<"AccAdd", 0, "add">;
-def OpenACC_ReductionOperatorMul : I32EnumAttrCase<"AccMul", 1, "mul">;
-def OpenACC_ReductionOperatorMax : I32EnumAttrCase<"AccMax", 2, "max">;
-def OpenACC_ReductionOperatorMin : I32EnumAttrCase<"AccMin", 3, "min">;
-def OpenACC_ReductionOperatorAnd : I32EnumAttrCase<"AccIand", 4, "iand">;
-def OpenACC_ReductionOperatorOr : I32EnumAttrCase<"AccIor", 5, "ior">;
-def OpenACC_ReductionOperatorXor : I32EnumAttrCase<"AccXor", 6, "xor">;
-def OpenACC_ReductionOperatorLogEqv : I32EnumAttrCase<"AccEqv", 7, "eqv">;
-def OpenACC_ReductionOperatorLogNeqv : I32EnumAttrCase<"AccNeqv", 8, "neqv">;
-def OpenACC_ReductionOperatorLogAnd : I32EnumAttrCase<"AccLand", 9, "land">;
-def OpenACC_ReductionOperatorLogOr : I32EnumAttrCase<"AccLor", 10, "lor">;
+def OpenACC_ReductionOperatorNone : I32EnumAttrCase<"AccNone", 0, "none">;
+def OpenACC_ReductionOperatorAdd : I32EnumAttrCase<"AccAdd", 1, "add">;
+def OpenACC_ReductionOperatorMul : I32EnumAttrCase<"AccMul", 2, "mul">;
+def OpenACC_ReductionOperatorMax : I32EnumAttrCase<"AccMax", 3, "max">;
+def OpenACC_ReductionOperatorMin : I32EnumAttrCase<"AccMin", 4, "min">;
+def OpenACC_ReductionOperatorAnd : I32EnumAttrCase<"AccIand", 5, "iand">;
+def OpenACC_ReductionOperatorOr : I32EnumAttrCase<"AccIor", 6, "ior">;
+def OpenACC_ReductionOperatorXor : I32EnumAttrCase<"AccXor", 7, "xor">;
+def OpenACC_ReductionOperatorLogEqv : I32EnumAttrCase<"AccEqv", 8, "eqv">;
+def OpenACC_ReductionOperatorLogNeqv : I32EnumAttrCase<"AccNeqv", 9, "neqv">;
+def OpenACC_ReductionOperatorLogAnd : I32EnumAttrCase<"AccLand", 10, "land">;
+def OpenACC_ReductionOperatorLogOr : I32EnumAttrCase<"AccLor", 11, "lor">;
def OpenACC_ReductionOperator : I32EnumAttr<"ReductionOperator",
"built-in reduction operations supported by OpenACC",
- [OpenACC_ReductionOperatorAdd, OpenACC_ReductionOperatorMul,
- OpenACC_ReductionOperatorMax, OpenACC_ReductionOperatorMin,
+ [OpenACC_ReductionOperatorNone, OpenACC_ReductionOperatorAdd,
+ OpenACC_ReductionOperatorMul, OpenACC_ReductionOperatorMax, OpenACC_ReductionOperatorMin,
OpenACC_ReductionOperatorAnd, OpenACC_ReductionOperatorOr,
OpenACC_ReductionOperatorXor, OpenACC_ReductionOperatorLogEqv,
OpenACC_ReductionOperatorLogNeqv, OpenACC_ReductionOperatorLogAnd,
More information about the flang-commits
mailing list