[flang-commits] [flang] [flang][OpenACC] simplify copy and combiner recipe generation (PR #164988)
via flang-commits
flang-commits at lists.llvm.org
Fri Oct 24 07:39:25 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-openacc
@llvm/pr-subscribers-flang-fir-hlfir
Author: None (jeanPerier)
<details>
<summary>Changes</summary>
Currently OpenACC copy and combiner recipe generation have non trivial ad-hoc handling of the FIR types, and ends-up generating hlfir.assign that become runtime calls for arrays because of the lack of aliasing guarantees given to FIR for the region arguments.
This code was started before HFLIR, and with HLFIR tools, most of the type specific handling can just be removed to use hlfir generic helper and assign (-230 lines in OpenACC.cpp).
To avoid ending up with runtime calls in recipes:
- use hlfir.assign temporary_lhs in copies to indicate that the copy cannot alias with the rhs.
- add a new genNoAliasAssignment hlfir helper that takes its logic from the HLFIRInlineIntrinsic pass and allows applying a scalar combiner before generating the scalar assignments. This allows generating loops directly for the reduction and avoid having to find a clever way to signal HLFIR that the arguments do not alias (an other option would have been to introduce a dummy_scope and declares, but that would not be enough for POINTERs, and I am not sure we should start using this as a noalias operation).
---
Patch is 191.81 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/164988.diff
6 Files Affected:
- (modified) flang/include/flang/Optimizer/Builder/HLFIRTools.h (+35)
- (modified) flang/lib/Lower/OpenACC.cpp (+122-355)
- (modified) flang/lib/Optimizer/Builder/HLFIRTools.cpp (+61)
- (modified) flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRAssign.cpp (+2-20)
- (modified) flang/test/Lower/OpenACC/acc-private.f90 (+18-35)
- (modified) flang/test/Lower/OpenACC/acc-reduction.f90 (+1200-751)
``````````diff
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 9f7c10c2b06c2..891373e8dbb0a 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -450,6 +450,41 @@ mlir::Value inlineElementalOp(
mlir::IRMapping &mapper,
const std::function<bool(hlfir::ElementalOp)> &mustRecursivelyInline);
+/// Generate an element-by-element assignment from \p rhs to \p lhs for arrays
+/// that are known not to alias. The assignment is performed using a loop nest
+/// over the optimal extents deduced from both shapes. If \p emitWorkshareLoop
+/// is true, a workshare loop construct may be emitted when available.
+/// Allocatable LHS must be allocated with the right shape and parameters.
+void genNoAliasArrayAssignment(
+ mlir::Location loc, fir::FirOpBuilder &builder, hlfir::Entity rhs,
+ hlfir::Entity lhs, bool emitWorkshareLoop = false,
+ bool temporaryLHS = false,
+ std::function<hlfir::Entity(mlir::Location, fir::FirOpBuilder &,
+ hlfir::Entity, hlfir::Entity)> *combiner =
+ nullptr);
+
+/// Generate an assignment from \p rhs to \p lhs when they are known not to
+/// alias. Handles both arrays and scalars: for arrays, delegates to
+/// genNoAliasArrayAssignment; for scalars, performs load/store for trivial
+/// scalar types and falls back to hlfir.assign otherwise.
+/// Allocatable LHS must be allocated with the right shape and parameters.
+void genNoAliasAssignment(
+ mlir::Location loc, fir::FirOpBuilder &builder, hlfir::Entity rhs,
+ hlfir::Entity lhs, bool emitWorkshareLoop = false,
+ bool temporaryLHS = false,
+ std::function<hlfir::Entity(mlir::Location, fir::FirOpBuilder &,
+ hlfir::Entity, hlfir::Entity)> *combiner =
+ nullptr);
+inline void genNoAliasAssignment(
+ mlir::Location loc, fir::FirOpBuilder &builder, hlfir::Entity rhs,
+ hlfir::Entity lhs, bool emitWorkshareLoop, bool temporaryLHS,
+ std::function<hlfir::Entity(mlir::Location, fir::FirOpBuilder &,
+ hlfir::Entity, hlfir::Entity)>
+ combiner) {
+ genNoAliasAssignment(loc, builder, rhs, lhs, emitWorkshareLoop, temporaryLHS,
+ &combiner);
+}
+
/// Create a new temporary with the shape and parameters of the provided
/// hlfir.eval_in_mem operation and clone the body of the hlfir.eval_in_mem
/// operating on this new temporary. returns the temporary and whether the
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index d7861ac6463c8..60401f41f5381 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1196,59 +1196,6 @@ genConstantBounds(fir::FirOpBuilder &builder, mlir::Location loc,
return {lb, ub, step};
}
-static mlir::Value genShapeFromBoundsOrArgs(
- mlir::Location loc, fir::FirOpBuilder &builder, fir::SequenceType seqTy,
- const llvm::SmallVector<mlir::Value> &bounds, mlir::ValueRange arguments) {
- llvm::SmallVector<mlir::Value> args;
- if (bounds.empty() && seqTy) {
- if (seqTy.hasDynamicExtents()) {
- assert(!arguments.empty() && "arguments must hold the entity");
- auto entity = hlfir::Entity{arguments[0]};
- return hlfir::genShape(loc, builder, entity);
- }
- return genShapeOp(builder, seqTy, loc).getResult();
- } else if (areAllBoundConstant(bounds)) {
- for (auto bound : llvm::reverse(bounds)) {
- auto dataBound =
- mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
- args.append(genConstantBounds(builder, loc, dataBound));
- }
- } else {
- assert(((arguments.size() - 2) / 3 == seqTy.getDimension()) &&
- "Expect 3 block arguments per dimension");
- for (auto arg : arguments.drop_front(2))
- args.push_back(arg);
- }
-
- assert(args.size() % 3 == 0 && "Triplets must be a multiple of 3");
- llvm::SmallVector<mlir::Value> extents;
- mlir::Type idxTy = builder.getIndexType();
- mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
- mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
- for (unsigned i = 0; i < args.size(); i += 3) {
- mlir::Value s1 =
- mlir::arith::SubIOp::create(builder, loc, args[i + 1], args[0]);
- mlir::Value s2 = mlir::arith::AddIOp::create(builder, loc, s1, one);
- mlir::Value s3 =
- mlir::arith::DivSIOp::create(builder, loc, s2, args[i + 2]);
- mlir::Value cmp = mlir::arith::CmpIOp::create(
- builder, loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
- mlir::Value ext =
- mlir::arith::SelectOp::create(builder, loc, cmp, s3, zero);
- extents.push_back(ext);
- }
- return fir::ShapeOp::create(builder, loc, extents);
-}
-
-static hlfir::DesignateOp::Subscripts
-getSubscriptsFromArgs(mlir::ValueRange args) {
- hlfir::DesignateOp::Subscripts triplets;
- for (unsigned i = 2; i < args.size(); i += 3)
- triplets.emplace_back(
- hlfir::DesignateOp::Triplet{args[i], args[i + 1], args[i + 2]});
- return triplets;
-}
-
static hlfir::Entity genDesignateWithTriplets(
fir::FirOpBuilder &builder, mlir::Location loc, hlfir::Entity &entity,
hlfir::DesignateOp::Subscripts &triplets, mlir::Value shape) {
@@ -1262,19 +1209,58 @@ static hlfir::Entity genDesignateWithTriplets(
return hlfir::Entity{designate.getResult()};
}
-mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
- fir::FirOpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc,
- mlir::Type ty, llvm::SmallVector<mlir::Value> &bounds) {
- mlir::ModuleOp mod =
- builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
- if (auto recipe =
- mod.lookupSymbol<mlir::acc::FirstprivateRecipeOp>(recipeName))
- return recipe;
+static std::pair<hlfir::Entity, hlfir::Entity>
+genArraySectionsInRecipe(fir::FirOpBuilder &builder, mlir::Location loc,
+ llvm::SmallVector<mlir::Value> &dataOperationBounds,
+ mlir::ValueRange recipeArguments,
+ bool allConstantBound, hlfir::Entity lhs,
+ hlfir::Entity rhs) {
+ // Get the list of lb,ub,step values for the sections that can be used inside
+ // the recipe region.
+ llvm::SmallVector<mlir::Value> bounds;
+ if (allConstantBound) {
+ // For constant bounds, the bounds are not region arguments. Materialize
+ // constants looking at the IR for the bounds on the data operation.
+ for (auto bound : llvm::reverse(dataOperationBounds)) {
+ auto dataBound =
+ mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
+ bounds.append(genConstantBounds(builder, loc, dataBound));
+ }
+ } else {
+ // If one bound is not constant, all of the bounds are region arguments.
+ for (auto arg : recipeArguments.drop_front(2))
+ bounds.push_back(arg);
+ }
+ // Compute the fir.shape of the array section and the triplets to create
+ // hlfir.designate.
+ assert(lhs.getRank() * 3 == static_cast<int>(bounds.size()) &&
+ "must get lb,ub,step for each dimension");
+ llvm::SmallVector<mlir::Value> extents;
+ mlir::Type idxTy = builder.getIndexType();
+ hlfir::DesignateOp::Subscripts triplets;
+ for (unsigned i = 0; i < bounds.size(); i += 3) {
+ extents.push_back(builder.genExtentFromTriplet(
+ loc, bounds[i], bounds[i + 1], bounds[i + 2], idxTy));
+ triplets.emplace_back(
+ hlfir::DesignateOp::Triplet{bounds[i], bounds[i + 1], bounds[i + 2]});
+ }
+ lhs = hlfir::derefPointersAndAllocatables(loc, builder, lhs);
+ rhs = hlfir::derefPointersAndAllocatables(loc, builder, rhs);
+ mlir::Value shape = fir::ShapeOp::create(builder, loc, extents);
+ hlfir::Entity leftSection =
+ genDesignateWithTriplets(builder, loc, lhs, triplets, shape);
+ hlfir::Entity rightSection =
+ genDesignateWithTriplets(builder, loc, rhs, triplets, shape);
+ return {leftSection, rightSection};
+}
- auto ip = builder.saveInsertionPoint();
- auto recipe = genRecipeOp<mlir::acc::FirstprivateRecipeOp>(
- builder, mod, recipeName, loc, ty);
- bool allConstantBound = areAllBoundConstant(bounds);
+// Generate the combiner or copy region block and block arguments and return the
+// source and destination entities.
+static std::pair<hlfir::Entity, hlfir::Entity>
+genRecipeCombinerOrCopyRegion(fir::FirOpBuilder &builder, mlir::Location loc,
+ mlir::Type ty, mlir::Region ®ion,
+ llvm::SmallVector<mlir::Value> &bounds,
+ bool allConstantBound) {
llvm::SmallVector<mlir::Type> argsTy{ty, ty};
llvm::SmallVector<mlir::Location> argsLoc{loc, loc};
if (!allConstantBound) {
@@ -1289,72 +1275,56 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
argsLoc.push_back(dataBound.getStartIdx().getLoc());
}
}
- builder.createBlock(&recipe.getCopyRegion(), recipe.getCopyRegion().end(),
- argsTy, argsLoc);
+ mlir::Block *block =
+ builder.createBlock(®ion, region.end(), argsTy, argsLoc);
+ builder.setInsertionPointToEnd(®ion.back());
+ return {hlfir::Entity{block->getArgument(0)},
+ hlfir::Entity{block->getArgument(1)}};
+}
- builder.setInsertionPointToEnd(&recipe.getCopyRegion().back());
- ty = fir::unwrapRefType(ty);
- if (fir::isa_trivial(ty)) {
- mlir::Value initValue = fir::LoadOp::create(
- builder, loc, recipe.getCopyRegion().front().getArgument(0));
- fir::StoreOp::create(builder, loc, initValue,
- recipe.getCopyRegion().front().getArgument(1));
- } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
- fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
- auto shape = genShapeFromBoundsOrArgs(
- loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
-
- auto leftDeclOp = hlfir::DeclareOp::create(
- builder, loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{},
- shape);
- auto rightDeclOp = hlfir::DeclareOp::create(
- builder, loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{},
- shape);
-
- hlfir::DesignateOp::Subscripts triplets =
- getSubscriptsFromArgs(recipe.getCopyRegion().getArguments());
- auto leftEntity = hlfir::Entity{leftDeclOp.getBase()};
- auto left =
- genDesignateWithTriplets(firBuilder, loc, leftEntity, triplets, shape);
- auto rightEntity = hlfir::Entity{rightDeclOp.getBase()};
- auto right =
- genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
-
- hlfir::AssignOp::create(firBuilder, loc, left, right);
-
- } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
- fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
- llvm::SmallVector<mlir::Value> tripletArgs;
- mlir::Type innerTy = fir::extractSequenceType(boxTy);
- fir::SequenceType seqTy =
- mlir::dyn_cast_or_null<fir::SequenceType>(innerTy);
- if (!seqTy)
- TODO(loc, "Unsupported boxed type in OpenACC firstprivate");
-
- auto shape = genShapeFromBoundsOrArgs(
- loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
- hlfir::DesignateOp::Subscripts triplets =
- getSubscriptsFromArgs(recipe.getCopyRegion().getArguments());
- auto leftEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(0)};
- auto left =
- genDesignateWithTriplets(firBuilder, loc, leftEntity, triplets, shape);
- auto rightEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(1)};
- auto right =
- genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
- hlfir::AssignOp::create(firBuilder, loc, left, right);
- } else {
- // Copy scalar derived type.
- // The temporary_lhs flag allows indicating that user defined assignments
- // should not be called while copying components, and that the LHS and RHS
- // are known to not alias since the LHS is a created object.
- hlfir::AssignOp::create(
- builder, loc, recipe.getCopyRegion().getArgument(0),
- recipe.getCopyRegion().getArgument(1), /*realloc=*/false,
- /*keep_lhs_length_if_realloc=*/false, /*temporary_lhs=*/true);
- }
+mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
+ fir::FirOpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc,
+ mlir::Type ty, llvm::SmallVector<mlir::Value> &bounds) {
+ mlir::ModuleOp mod =
+ builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
+ if (auto recipe =
+ mod.lookupSymbol<mlir::acc::FirstprivateRecipeOp>(recipeName))
+ return recipe;
+
+ mlir::OpBuilder::InsertionGuard guard(builder);
+ auto recipe = genRecipeOp<mlir::acc::FirstprivateRecipeOp>(
+ builder, mod, recipeName, loc, ty);
+ bool allConstantBound = areAllBoundConstant(bounds);
+ auto [source, destination] = genRecipeCombinerOrCopyRegion(
+ builder, loc, ty, recipe.getCopyRegion(), bounds, allConstantBound);
+
+ fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
+
+ source = hlfir::derefPointersAndAllocatables(loc, builder, source);
+ destination = hlfir::derefPointersAndAllocatables(loc, builder, destination);
+ if (!bounds.empty())
+ std::tie(source, destination) = genArraySectionsInRecipe(
+ firBuilder, loc, bounds, recipe.getCopyRegion().getArguments(),
+ allConstantBound, source, destination);
+ // The source and the destination of the firstprivate copy cannot alias,
+ // the destination is already properly allocated, so a simple assignment
+ // can be generated right away to avoid ending-up with runtime calls
+ // for arrays of numerical, logical and, character types.
+ //
+ // The temporary_lhs flag allows indicating that user defined assignments
+ // should not be called while copying components, and that the LHS and RHS
+ // are known to not alias since the LHS is a created object.
+ //
+ // TODO: detect cases where user defined assignment is needed and add a TODO.
+ // using temporary_lhs allows more aggressive optimizations of simple derived
+ // types. Existing compilers supporting OpenACC do not call user defined
+ // assignments, some use case is needed to decide what to do.
+ source = hlfir::loadTrivialScalar(loc, builder, source);
+ hlfir::AssignOp::create(builder, loc, source, destination, /*realloc=*/false,
+ /*keep_lhs_length_if_realloc=*/false,
+ /*temporary_lhs=*/true);
mlir::acc::TerminatorOp::create(builder, loc);
- builder.restoreInsertionPoint(ip);
return recipe;
}
@@ -1611,205 +1581,6 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
TODO(loc, "reduction operator");
}
-static hlfir::DesignateOp::Subscripts
-getTripletsFromArgs(mlir::acc::ReductionRecipeOp recipe) {
- hlfir::DesignateOp::Subscripts triplets;
- for (unsigned i = 2; i < recipe.getCombinerRegion().getArguments().size();
- i += 3)
- triplets.emplace_back(hlfir::DesignateOp::Triplet{
- recipe.getCombinerRegion().getArgument(i),
- recipe.getCombinerRegion().getArgument(i + 1),
- recipe.getCombinerRegion().getArgument(i + 2)});
- return triplets;
-}
-
-static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
- mlir::acc::ReductionOperator op, mlir::Type ty,
- mlir::Value value1, mlir::Value value2,
- mlir::acc::ReductionRecipeOp &recipe,
- llvm::SmallVector<mlir::Value> &bounds,
- bool allConstantBound) {
- ty = fir::unwrapRefType(ty);
-
- if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty)) {
- mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
- llvm::SmallVector<fir::DoLoopOp> loops;
- llvm::SmallVector<mlir::Value> ivs;
- if (seqTy.hasDynamicExtents()) {
- auto shape =
- genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
- recipe.getCombinerRegion().getArguments());
- auto v1DeclareOp = hlfir::DeclareOp::create(builder, loc, value1,
- llvm::StringRef{}, shape);
- auto v2DeclareOp = hlfir::DeclareOp::create(builder, loc, value2,
- llvm::StringRef{}, shape);
- hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
-
- llvm::SmallVector<mlir::Value> lenParamsLeft;
- auto leftEntity = hlfir::Entity{v1DeclareOp.getBase()};
- hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
- auto leftDesignate = hlfir::DesignateOp::create(
- builder, loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
- /*component=*/"",
- /*componentShape=*/mlir::Value{}, triplets,
- /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
- shape, lenParamsLeft);
- auto left = hlfir::Entity{leftDesignate.getResult()};
-
- llvm::SmallVector<mlir::Value> lenParamsRight;
- auto rightEntity = hlfir::Entity{v2DeclareOp.getBase()};
- hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsLeft);
- auto rightDesignate = hlfir::DesignateOp::create(
- builder, loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
- /*component=*/"",
- /*componentShape=*/mlir::Value{}, triplets,
- /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
- shape, lenParamsRight);
- auto right = hlfir::Entity{rightDesignate.getResult()};
-
- llvm::SmallVector<mlir::Value, 1> typeParams;
- auto genKernel = [&builder, &loc, op, seqTy, &left, &right](
- mlir::Location l, fir::FirOpBuilder &b,
- mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
- auto leftElement = hlfir::getElementAt(l, b, left, oneBasedIndices);
- auto rightElement = hlfir::getElementAt(l, b, right, oneBasedIndices);
- auto leftVal = hlfir::loadTrivialScalar(l, b, leftElement);
- auto rightVal = hlfir::loadTrivialScalar(l, b, rightElement);
- return hlfir::Entity{genScalarCombiner(
- builder, loc, op, seqTy.getEleTy(), leftVal, rightVal)};
- };
- mlir::Value elemental = hlfir::genElementalOp(
- loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
- /*isUnordered=*/true);
- hlfir::AssignOp::create(builder, loc, elemental, v1DeclareOp.getBase());
- return;
- }
- if (bounds.empty()) {
- llvm::SmallVector<mlir::Value> extents;
- mlir::Type idxTy = builder.getIndexType();
- for (auto extent : llvm::reverse(seqTy.getShape())) {
- mlir::Value lb = mlir::arith::ConstantOp::create(
- builder, loc, idxTy, builder.getIntegerAttr(idxTy, 0));
- mlir::Value ub = mlir::arith::ConstantOp::create(
- builder, loc, idxTy, builder.getIntegerAttr(idxTy, extent - 1));
- mlir::Value step = mlir::arith::ConstantOp::create(
- builder, loc, idxTy, builder.getIntegerAttr(idxTy, 1));
- auto loop = fir::DoLoopOp::create(builder, loc, lb, ub, step,
- /*unordered=*/false);
- builder.setInsertionPointToStart(loop.getBody());
- loops.push_back(loop);
- ivs.push_back(loop.getInductionVar());
- }
- } else if (allConstantBound) {
- // Use the constant bound directly in the combiner region so they do not
- // need to be passed as block argument.
- assert(!bounds.empty() &&
- "seq type with constant bounds cannot have empty bounds");
- for (auto bound : llvm::reverse(bounds)) {
- auto dataBound =
- mlir::dyn_cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
- llvm::SmallVector<mlir::Value> values =
- genConstantBounds(builder, loc, dataBound);
- auto loop =
- fir::DoLoopOp::create(builder, loc, values[0], values[1], values[2],
- /*unordered=*/false);
- builder.setInsertionPointToStart(loop.getBody());
- loops.push_back(loop);
- ivs.push_back(lo...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/164988
More information about the flang-commits
mailing list