[flang-commits] [flang] 0d40574 - [flang] Inline hlfir.copy_in for trivial types (#138718)
via flang-commits
flang-commits at lists.llvm.org
Fri Jun 6 07:10:20 PDT 2025
Author: Kajetan Puchalski
Date: 2025-06-06T15:10:17+01:00
New Revision: 0d40574e16e8b192e0b3e4caa313b42cbfb88821
URL: https://github.com/llvm/llvm-project/commit/0d40574e16e8b192e0b3e4caa313b42cbfb88821
DIFF: https://github.com/llvm/llvm-project/commit/0d40574e16e8b192e0b3e4caa313b42cbfb88821.diff
LOG: [flang] Inline hlfir.copy_in for trivial types (#138718)
hlfir.copy_in implements copying non-contiguous array slices for
functions that take in arrays required to be contiguous through
flang-rt.
For large arrays of trivial types, this can incur overhead compared to a
plain, inlined copy loop.
To address that, add a new InlineHLFIRCopyIn optimisation pass to inline
hlfir.copy_in operations for trivial types.
For the time being, the pattern is only applied in cases where the
copy-in does not require a corresponding copy-out, such as when the
function being called declares the array parameter as intent(in).
Applying this optimisation reduces the runtime of thornado-mini's
DeleptonizationProblem by about 10%.
---------
Signed-off-by: Kajetan Puchalski <kajetan.puchalski at arm.com>
Added:
flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
flang/test/HLFIR/inline-hlfir-copy-in.fir
Modified:
flang/include/flang/Optimizer/Builder/HLFIRTools.h
flang/include/flang/Optimizer/HLFIR/Passes.td
flang/lib/Optimizer/Builder/HLFIRTools.cpp
flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
flang/lib/Optimizer/Passes/Pipelines.cpp
Removed:
################################################################################
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index ed00cec04dc39..2cbad6e268a38 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -374,12 +374,14 @@ struct LoopNest {
/// loop constructs currently.
LoopNest genLoopNest(mlir::Location loc, fir::FirOpBuilder &builder,
mlir::ValueRange extents, bool isUnordered = false,
- bool emitWorkshareLoop = false);
+ bool emitWorkshareLoop = false,
+ bool couldVectorize = true);
inline LoopNest genLoopNest(mlir::Location loc, fir::FirOpBuilder &builder,
mlir::Value shape, bool isUnordered = false,
- bool emitWorkshareLoop = false) {
+ bool emitWorkshareLoop = false,
+ bool couldVectorize = true) {
return genLoopNest(loc, builder, getIndexExtents(loc, builder, shape),
- isUnordered, emitWorkshareLoop);
+ isUnordered, emitWorkshareLoop, couldVectorize);
}
/// The type of a callback that generates the body of a reduction
diff --git a/flang/include/flang/Optimizer/HLFIR/Passes.td b/flang/include/flang/Optimizer/HLFIR/Passes.td
index d445140118073..04d7aec5fe489 100644
--- a/flang/include/flang/Optimizer/HLFIR/Passes.td
+++ b/flang/include/flang/Optimizer/HLFIR/Passes.td
@@ -69,6 +69,10 @@ def InlineHLFIRAssign : Pass<"inline-hlfir-assign"> {
let summary = "Inline hlfir.assign operations";
}
+def InlineHLFIRCopyIn : Pass<"inline-hlfir-copy-in"> {
+ let summary = "Inline hlfir.copy_in operations";
+}
+
def PropagateFortranVariableAttributes : Pass<"propagate-fortran-attrs"> {
let summary = "Propagate FortranVariableFlagsAttr attributes through HLFIR";
}
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index ce82cad153893..b54cccbef14fa 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -21,6 +21,7 @@
#include "mlir/IR/IRMapping.h"
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/TypeSwitch.h"
+#include <mlir/Dialect/LLVMIR/LLVMAttrs.h>
#include <mlir/Dialect/OpenMP/OpenMPDialect.h>
#include <optional>
@@ -932,7 +933,8 @@ mlir::Value hlfir::inlineElementalOp(
hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
fir::FirOpBuilder &builder,
mlir::ValueRange extents, bool isUnordered,
- bool emitWorkshareLoop) {
+ bool emitWorkshareLoop,
+ bool couldVectorize) {
emitWorkshareLoop = emitWorkshareLoop && isUnordered;
hlfir::LoopNest loopNest;
assert(!extents.empty() && "must have at least one extent");
@@ -967,6 +969,15 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc,
auto ub = builder.createConvert(loc, indexType, extent);
auto doLoop =
builder.create<fir::DoLoopOp>(loc, one, ub, one, isUnordered);
+ if (!couldVectorize) {
+ mlir::LLVM::LoopVectorizeAttr va{mlir::LLVM::LoopVectorizeAttr::get(
+ builder.getContext(),
+ /*disable=*/builder.getBoolAttr(true), {}, {}, {}, {}, {}, {})};
+ mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get(
+ builder.getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ {},
+ /*unroll_and_jam*/ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});
+ doLoop.setLoopAnnotationAttr(la);
+ }
loopNest.body = doLoop.getBody();
builder.setInsertionPointToStart(loopNest.body);
// Reverse the indices so they are in column-major order.
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
index d959428ebd203..cc74273d9c5d9 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
@@ -5,6 +5,7 @@ add_flang_library(HLFIRTransforms
ConvertToFIR.cpp
InlineElementals.cpp
InlineHLFIRAssign.cpp
+ InlineHLFIRCopyIn.cpp
LowerHLFIRIntrinsics.cpp
LowerHLFIROrderedAssignments.cpp
ScheduleOrderedAssignments.cpp
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
new file mode 100644
index 0000000000000..7e8acc515ee26
--- /dev/null
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopyIn.cpp
@@ -0,0 +1,183 @@
+//===- InlineHLFIRCopyIn.cpp - Inline hlfir.copy_in ops -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Transform hlfir.copy_in array operations into loop nests performing element
+// per element assignments. For simplicity, the inlining is done for trivial
+// data types when the copy_in does not require a corresponding copy_out and
+// when the input array is not behind a pointer. This may change in the future.
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/OpenMP/Passes.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace hlfir {
+#define GEN_PASS_DEF_INLINEHLFIRCOPYIN
+#include "flang/Optimizer/HLFIR/Passes.h.inc"
+} // namespace hlfir
+
+#define DEBUG_TYPE "inline-hlfir-copy-in"
+
+static llvm::cl::opt<bool> noInlineHLFIRCopyIn(
+ "no-inline-hlfir-copy-in",
+ llvm::cl::desc("Do not inline hlfir.copy_in operations"),
+ llvm::cl::init(false));
+
+namespace {
+class InlineCopyInConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
+public:
+ using mlir::OpRewritePattern<hlfir::CopyInOp>::OpRewritePattern;
+
+ llvm::LogicalResult
+ matchAndRewrite(hlfir::CopyInOp copyIn,
+ mlir::PatternRewriter &rewriter) const override;
+};
+
+llvm::LogicalResult
+InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
+ mlir::PatternRewriter &rewriter) const {
+ fir::FirOpBuilder builder(rewriter, copyIn.getOperation());
+ mlir::Location loc = copyIn.getLoc();
+ hlfir::Entity inputVariable{copyIn.getVar()};
+ mlir::Type resultAddrType = copyIn.getCopiedIn().getType();
+ if (!fir::isa_trivial(inputVariable.getFortranElementType()))
+ return rewriter.notifyMatchFailure(copyIn,
+ "CopyInOp's data type is not trivial");
+
+ // There should be exactly one user of WasCopied - the corresponding
+ // CopyOutOp.
+ if (!copyIn.getWasCopied().hasOneUse())
+ return rewriter.notifyMatchFailure(
+ copyIn, "CopyInOp's WasCopied has no single user");
+ // The copy out should always be present, either to actually copy or just
+ // deallocate memory.
+ auto copyOut = mlir::dyn_cast<hlfir::CopyOutOp>(
+ copyIn.getWasCopied().user_begin().getCurrent().getUser());
+
+ if (!copyOut)
+ return rewriter.notifyMatchFailure(copyIn,
+ "CopyInOp has no direct CopyOut");
+
+ if (mlir::cast<fir::BaseBoxType>(resultAddrType).isAssumedRank())
+ return rewriter.notifyMatchFailure(copyIn,
+ "The result array is assumed-rank");
+
+ // Only inline the copy_in when copy_out does not need to be done, i.e. in
+ // case of intent(in).
+ if (copyOut.getVar())
+ return rewriter.notifyMatchFailure(copyIn, "CopyIn needs a copy-out");
+
+ inputVariable =
+ hlfir::derefPointersAndAllocatables(loc, builder, inputVariable);
+ mlir::Type sequenceType =
+ hlfir::getFortranElementOrSequenceType(inputVariable.getType());
+ fir::BoxType resultBoxType = fir::BoxType::get(sequenceType);
+ mlir::Value isContiguous =
+ builder.create<fir::IsContiguousBoxOp>(loc, inputVariable);
+ mlir::Operation::result_range results =
+ builder
+ .genIfOp(loc, {resultBoxType, builder.getI1Type()}, isContiguous,
+ /*withElseRegion=*/true)
+ .genThen([&]() {
+ mlir::Value result = inputVariable;
+ if (fir::isPointerType(inputVariable.getType())) {
+ result = builder.create<fir::ReboxOp>(
+ loc, resultBoxType, inputVariable, mlir::Value{},
+ mlir::Value{});
+ }
+ builder.create<fir::ResultOp>(
+ loc, mlir::ValueRange{result, builder.createBool(loc, false)});
+ })
+ .genElse([&] {
+ mlir::Value shape = hlfir::genShape(loc, builder, inputVariable);
+ llvm::SmallVector<mlir::Value> extents =
+ hlfir::getIndexExtents(loc, builder, shape);
+ llvm::StringRef tmpName{".tmp.copy_in"};
+ llvm::SmallVector<mlir::Value> lenParams;
+ mlir::Value alloc = builder.createHeapTemporary(
+ loc, sequenceType, tmpName, extents, lenParams);
+
+ auto declareOp = builder.create<hlfir::DeclareOp>(
+ loc, alloc, tmpName, shape, lenParams,
+ /*dummy_scope=*/nullptr);
+ hlfir::Entity temp{declareOp.getBase()};
+ hlfir::LoopNest loopNest =
+ hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+ flangomp::shouldUseWorkshareLowering(copyIn),
+ /*couldVectorize=*/false);
+ builder.setInsertionPointToStart(loopNest.body);
+ hlfir::Entity elem = hlfir::getElementAt(
+ loc, builder, inputVariable, loopNest.oneBasedIndices);
+ elem = hlfir::loadTrivialScalar(loc, builder, elem);
+ hlfir::Entity tempElem = hlfir::getElementAt(
+ loc, builder, temp, loopNest.oneBasedIndices);
+ builder.create<hlfir::AssignOp>(loc, elem, tempElem);
+ builder.setInsertionPointAfter(loopNest.outerOp);
+
+ mlir::Value result;
+ // Make sure the result is always a boxed array by boxing it
+ // ourselves if need be.
+ if (mlir::isa<fir::BaseBoxType>(temp.getType())) {
+ result = temp;
+ } else {
+ fir::ReferenceType refTy =
+ fir::ReferenceType::get(temp.getElementOrSequenceType());
+ mlir::Value refVal = builder.createConvert(loc, refTy, temp);
+ result = builder.create<fir::EmboxOp>(loc, resultBoxType, refVal,
+ shape);
+ }
+
+ builder.create<fir::ResultOp>(
+ loc, mlir::ValueRange{result, builder.createBool(loc, true)});
+ })
+ .getResults();
+
+ mlir::OpResult resultBox = results[0];
+ mlir::OpResult needsCleanup = results[1];
+
+ // Prepare the corresponding copyOut to free the temporary if it is required
+ auto alloca = builder.create<fir::AllocaOp>(loc, resultBox.getType());
+ auto store = builder.create<fir::StoreOp>(loc, resultBox, alloca);
+ rewriter.startOpModification(copyOut);
+ copyOut->setOperand(0, store.getMemref());
+ copyOut->setOperand(1, needsCleanup);
+ rewriter.finalizeOpModification(copyOut);
+
+ rewriter.replaceOp(copyIn, {resultBox, builder.genNot(loc, isContiguous)});
+ return mlir::success();
+}
+
+class InlineHLFIRCopyInPass
+ : public hlfir::impl::InlineHLFIRCopyInBase<InlineHLFIRCopyInPass> {
+public:
+ void runOnOperation() override {
+ mlir::MLIRContext *context = &getContext();
+
+ mlir::GreedyRewriteConfig config;
+ // Prevent the pattern driver from merging blocks.
+ config.setRegionSimplificationLevel(
+ mlir::GreedySimplifyRegionLevel::Disabled);
+
+ mlir::RewritePatternSet patterns(context);
+ if (!noInlineHLFIRCopyIn) {
+ patterns.insert<InlineCopyInConversion>(context);
+ }
+
+ if (mlir::failed(mlir::applyPatternsGreedily(
+ getOperation(), std::move(patterns), config))) {
+ mlir::emitError(getOperation()->getLoc(),
+ "failure in hlfir.copy_in inlining");
+ signalPassFailure();
+ }
+ }
+};
+} // namespace
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 00248032daf92..c28c8dd1730b0 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -260,6 +260,11 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
pm, hlfir::createOptimizedBufferization);
addNestedPassToAllTopLevelOperations<PassConstructor>(
pm, hlfir::createInlineHLFIRAssign);
+
+ if (optLevel == llvm::OptimizationLevel::O3) {
+ addNestedPassToAllTopLevelOperations<PassConstructor>(
+ pm, hlfir::createInlineHLFIRCopyIn);
+ }
}
pm.addPass(hlfir::createLowerHLFIROrderedAssignments());
pm.addPass(hlfir::createLowerHLFIRIntrinsics());
diff --git a/flang/test/HLFIR/inline-hlfir-copy-in.fir b/flang/test/HLFIR/inline-hlfir-copy-in.fir
new file mode 100644
index 0000000000000..f3c4b38962a0c
--- /dev/null
+++ b/flang/test/HLFIR/inline-hlfir-copy-in.fir
@@ -0,0 +1,206 @@
+// Test inlining of hlfir.copy_in
+// RUN: fir-opt --inline-hlfir-copy-in %s | FileCheck %s
+
+// Test inlining of hlfir.copy_in that does not require the array to be copied out
+func.func private @_test_inline_copy_in(%arg0: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "i"}, %arg2: !fir.ref<i32> {fir.bindc_name = "j"}) {
+ %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+ %1 = fir.dummy_scope : !fir.dscope
+ %2:2 = hlfir.declare %arg1 dummy_scope %1 {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+ %3:2 = hlfir.declare %arg2 dummy_scope %1 {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+ %4:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+ %5 = fir.load %2#0 : !fir.ref<i32>
+ %6 = fir.convert %5 : (i32) -> i64
+ %c1 = arith.constant 1 : index
+ %c1_0 = arith.constant 1 : index
+ %7:3 = fir.box_dims %4#1, %c1_0 : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+ %c1_1 = arith.constant 1 : index
+ %c0 = arith.constant 0 : index
+ %8 = arith.subi %7#1, %c1 : index
+ %9 = arith.addi %8, %c1_1 : index
+ %10 = arith.divsi %9, %c1_1 : index
+ %11 = arith.cmpi sgt, %10, %c0 : index
+ %12 = arith.select %11, %10, %c0 : index
+ %13 = fir.load %3#0 : !fir.ref<i32>
+ %14 = fir.convert %13 : (i32) -> i64
+ %15 = fir.shape %12 : (index) -> !fir.shape<1>
+ %16 = hlfir.designate %4#0 (%6, %c1:%7#1:%c1_1, %14) shape %15 : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+ %c100_i32 = arith.constant 100 : i32
+ %17:2 = hlfir.copy_in %16 to %0 : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+ %18 = fir.box_addr %17#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+ %19:3 = hlfir.associate %c100_i32 {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+ fir.call @_QFPsb(%18, %19#0) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+ hlfir.copy_out %0, %17#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1) -> ()
+ hlfir.end_associate %19#1, %19#2 : !fir.ref<i32>, i1
+ return
+}
+
+// CHECK: #loop_vectorize = #llvm.loop_vectorize<disable = true>
+// CHECK: #loop_annotation = #llvm.loop_annotation<vectorize = #loop_vectorize>
+// CHECK-LABEL: func.func private @_test_inline_copy_in(
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"},
+// CHECK-SAME: %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
+// CHECK-SAME: %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "j"}) {
+// CHECK: %[[VAL_3:.*]] = arith.constant true
+// CHECK: %[[VAL_4:.*]] = arith.constant false
+// CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32
+// CHECK: %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK: %[[VAL_7:.*]] = arith.constant 1 : index
+// CHECK: %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK: %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_1:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %[[VAL_8:.*]] {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+// CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_22:.*]]#0 : !fir.ref<i32>
+// CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11:.*]] : (i32) -> i64
+// CHECK: %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_10:.*]]#1, %[[VAL_7:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+// CHECK: %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13:.*]]#1, %[[VAL_6:.*]] : index
+// CHECK: %[[VAL_15:.*]] = arith.select %[[VAL_14:.*]], %[[VAL_13:.*]]#1, %[[VAL_6:.*]] : index
+// CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_9:.*]]#0 : !fir.ref<i32>
+// CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16:.*]] : (i32) -> i64
+// CHECK: %[[VAL_18:.*]] = fir.shape %[[VAL_15:.*]] : (index) -> !fir.shape<1>
+// CHECK: %[[VAL_19:.*]] = hlfir.designate %[[VAL_10:.*]]#0 (%[[VAL_12:.*]], %[[VAL_7:.*]]:%[[VAL_13:.*]]#1:%[[VAL_7:.*]], %[[VAL_17:.*]]) shape %[[VAL_18:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+// CHECK: %[[VAL_20:.*]] = fir.is_contiguous_box %[[VAL_19:.*]] whole : (!fir.box<!fir.array<?xf64>>) -> i1
+// CHECK: %[[VAL_21:.*]]:2 = fir.if %[[VAL_20:.*]] -> (!fir.box<!fir.array<?xf64>>, i1) {
+// CHECK: fir.result %[[VAL_19:.*]], %[[VAL_4:.*]] : !fir.box<!fir.array<?xf64>>, i1
+// CHECK: } else {
+// CHECK: %[[VAL_24:.*]] = fir.allocmem !fir.array<?xf64>, %[[VAL_15:.*]] {bindc_name = ".tmp.copy_in", uniq_name = ""}
+// CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_24:.*]](%[[VAL_18:.*]]) {uniq_name = ".tmp.copy_in"} : (!fir.heap<!fir.array<?xf64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.heap<!fir.array<?xf64>>)
+// CHECK: fir.do_loop %arg3 = %[[VAL_7:.*]] to %[[VAL_15:.*]] step %[[VAL_7:.*]] unordered attributes {loopAnnotation = #loop_annotation} {
+// CHECK: %[[VAL_26:.*]] = hlfir.designate %[[VAL_19:.*]] (%arg3) : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK: %[[VAL_27:.*]] = fir.load %[[VAL_26:.*]] : !fir.ref<f64>
+// CHECK: %[[VAL_28:.*]] = hlfir.designate %[[VAL_25:.*]]#0 (%arg3) : (!fir.box<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK: hlfir.assign %[[VAL_27:.*]] to %[[VAL_28:.*]] : f64, !fir.ref<f64>
+// CHECK: }
+// CHECK: fir.result %[[VAL_25:.*]]#0, %[[VAL_3:.*]] : !fir.box<!fir.array<?xf64>>, i1
+// CHECK: }
+// CHECK: %[[VAL_22:.*]] = fir.box_addr %[[VAL_21:.*]]#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+// CHECK: %[[VAL_23:.*]]:3 = hlfir.associate %[[VAL_5:.*]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+// CHECK: fir.call @_QFPsb(%[[VAL_22:.*]], %[[VAL_23:.*]]#0) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+// CHECK: hlfir.copy_out %16, %15#1 : (!fir.ref<!fir.box<!fir.array<?xf64>>>, i1) -> ()
+// CHECK: hlfir.end_associate %[[VAL_23:.*]]#1, %[[VAL_23:.*]]#2 : !fir.ref<i32>, i1
+// CHECK: return
+// CHECK: }
+
+// Test not inlining of hlfir.copy_in that requires the array to be copied out
+func.func private @_test_no_inline_copy_in(%arg0: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "i"}, %arg2: !fir.ref<i32> {fir.bindc_name = "j"}) {
+ %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+ %1 = fir.dummy_scope : !fir.dscope
+ %2:2 = hlfir.declare %arg1 dummy_scope %1 {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+ %3:2 = hlfir.declare %arg2 dummy_scope %1 {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+ %4:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+ %5 = fir.load %2#0 : !fir.ref<i32>
+ %6 = fir.convert %5 : (i32) -> i64
+ %c1 = arith.constant 1 : index
+ %c1_0 = arith.constant 1 : index
+ %7:3 = fir.box_dims %4#1, %c1_0 : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+ %c1_1 = arith.constant 1 : index
+ %c0 = arith.constant 0 : index
+ %8 = arith.subi %7#1, %c1 : index
+ %9 = arith.addi %8, %c1_1 : index
+ %10 = arith.divsi %9, %c1_1 : index
+ %11 = arith.cmpi sgt, %10, %c0 : index
+ %12 = arith.select %11, %10, %c0 : index
+ %13 = fir.load %3#0 : !fir.ref<i32>
+ %14 = fir.convert %13 : (i32) -> i64
+ %15 = fir.shape %12 : (index) -> !fir.shape<1>
+ %16 = hlfir.designate %4#0 (%6, %c1:%7#1:%c1_1, %14) shape %15 : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+ %c100_i32 = arith.constant 100 : i32
+ %17:2 = hlfir.copy_in %16 to %0 : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+ %18 = fir.box_addr %17#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+ %19:3 = hlfir.associate %c100_i32 {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+ fir.call @_QFPsb(%18, %19#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+ hlfir.copy_out %0, %17#1 to %16 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1, !fir.box<!fir.array<?xf64>>) -> ()
+ hlfir.end_associate %19#1, %19#2 : !fir.ref<i32>, i1
+ return
+}
+
+// CHECK-LABEL: func.func private @_test_no_inline_copy_in(
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf64>> {fir.bindc_name = "x"},
+// CHECK-SAME: %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
+// CHECK-SAME: %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "j"}) {
+// CHECK: %[[VAL_3:.*]] = arith.constant 100 : i32
+// CHECK: %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK: %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK: %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf64>>>
+// CHECK: %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK: %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ei"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ej"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %[[VAL_7:.*]] {uniq_name = "_QFFsb2Ex"} : (!fir.box<!fir.array<?x?x?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf64>>, !fir.box<!fir.array<?x?x?xf64>>)
+// CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_8:.*]]#0 : !fir.ref<i32>
+// CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_11:.*]] : (i32) -> i64
+// CHECK: %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_10:.*]]#1, %[[VAL_5:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, index) -> (index, index, index)
+// CHECK: %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13:.*]]#1, %[[VAL_4:.*]] : index
+// CHECK: %[[VAL_15:.*]] = arith.select %[[VAL_14:.*]], %[[VAL_13:.*]]#1, %[[VAL_4:.*]] : index
+// CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_9:.*]]#0 : !fir.ref<i32>
+// CHECK: %[[VAL_17:.*]] = fir.convert %[[VAL_16:.*]] : (i32) -> i64
+// CHECK: %[[VAL_18:.*]] = fir.shape %[[VAL_15:.*]] : (index) -> !fir.shape<1>
+// CHECK: %[[VAL_19:.*]] = hlfir.designate %[[VAL_10:.*]]#0 (%[[VAL_12:.*]], %[[VAL_5:.*]]:%[[VAL_13:.*]]#1:%[[VAL_5:.*]], %[[VAL_17:.*]]) shape %[[VAL_18:.*]] : (!fir.box<!fir.array<?x?x?xf64>>, i64, index, index, index, i64, !fir.shape<1>) -> !fir.box<!fir.array<?xf64>>
+// CHECK: %[[VAL_20:.*]]:2 = hlfir.copy_in %[[VAL_19:.*]] to %[[VAL_6:.*]] : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>) -> (!fir.box<!fir.array<?xf64>>, i1)
+// CHECK: %[[VAL_21:.*]] = fir.box_addr %[[VAL_20:.*]]#0 : (!fir.box<!fir.array<?xf64>>) -> !fir.ref<!fir.array<?xf64>>
+// CHECK: %[[VAL_22:.*]]:3 = hlfir.associate %[[VAL_3:.*]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+// CHECK: fir.call @_QFPsb(%[[VAL_21:.*]], %[[VAL_22:.*]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xf64>>, !fir.ref<i32>) -> ()
+// CHECK: hlfir.copy_out %[[VAL_6:.*]], %[[VAL_20:.*]]#1 to %[[VAL_19:.*]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, i1, !fir.box<!fir.array<?xf64>>) -> ()
+// CHECK: hlfir.end_associate %[[VAL_22:.*]]#1, %[[VAL_22:.*]]#2 : !fir.ref<i32>, i1
+// CHECK: return
+// CHECK: }
+
+// Test not inlining optional dummy arguments (no direct copy-out)
+func.func @_QPoptional_copy_in_out(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
+ %false = arith.constant false
+ %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+ %1 = fir.dummy_scope : !fir.dscope
+ %2:2 = hlfir.declare %arg0 dummy_scope %1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+ %3 = fir.is_present %2#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+ %4:2 = fir.if %3 -> (!fir.ref<!fir.array<?xf32>>, i1) {
+ %5:2 = hlfir.copy_in %2#0 to %0 : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.box<!fir.array<?xf32>>, i1)
+ %6 = fir.box_addr %5#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+ fir.result %6, %5#1 : !fir.ref<!fir.array<?xf32>>, i1
+ } else {
+ %5 = fir.absent !fir.ref<!fir.array<?xf32>>
+ fir.result %5, %false : !fir.ref<!fir.array<?xf32>>, i1
+ }
+ fir.call @_QPtakes_optional_explicit(%4#0) fastmath<contract> : (!fir.ref<!fir.array<?xf32>>) -> ()
+ hlfir.copy_out %0, %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, i1) -> ()
+ return
+}
+
+// CHECK-LABEL: func.func @_QPoptional_copy_in_out(
+// CHECK-SAME: %[[ARG_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
+// CHECK: %false = arith.constant false
+// CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>>
+// CHECK: %[[VAL_1:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG_0:.*]] dummy_scope %[[VAL_1:.*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK: %[[VAL_3:.*]] = fir.is_present %[[VAL_2:.*]]#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+// CHECK: %[[VAL_4:.*]]:2 = fir.if %[[VAL_3:.*]] -> (!fir.ref<!fir.array<?xf32>>, i1) {
+// CHECK: %[[VAL_5:.*]]:2 = hlfir.copy_in %[[VAL_2:.*]]#0 to %[[VAL_0:.*]] : (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.box<!fir.array<?xf32>>, i1)
+// CHECK: %[[VAL_6:.*]] = fir.box_addr %[[VAL_5:.*]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+// CHECK: fir.result %[[VAL_6:.*]], %[[VAL_5:.*]]#1 : !fir.ref<!fir.array<?xf32>>, i1
+// CHECK: } else {
+// CHECK: %[[VAL_5:.*]] = fir.absent !fir.ref<!fir.array<?xf32>>
+// CHECK: fir.result %[[VAL_5:.*]], %false : !fir.ref<!fir.array<?xf32>>, i1
+// CHECK: }
+// CHECK: fir.call @_QPtakes_optional_explicit(%[[VAL_4:.*]]#0) fastmath<contract> : (!fir.ref<!fir.array<?xf32>>) -> ()
+// CHECK: hlfir.copy_out %[[VAL_0:.*]], %[[VAL_4:.*]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, i1) -> ()
+// CHECK: return
+// CHECK: }
+
+// Test not inlining of assumed-rank arrays
+func.func @_QPtest_copy_in_out_2(%arg0: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+ %0 = fir.alloca !fir.box<!fir.heap<!fir.array<*:f32>>>
+ %1 = fir.dummy_scope : !fir.dscope
+ %2:2 = hlfir.declare %arg0 dummy_scope %1 {uniq_name = "_QFtest_copy_in_out_2Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+ %3:2 = hlfir.copy_in %2#0 to %0 : (!fir.box<!fir.array<*:f32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.box<!fir.array<*:f32>>, i1)
+ fir.call @_QPtakes_contiguous_intentin(%3#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
+ hlfir.copy_out %0, %3#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, i1) -> ()
+ return
+}
+
+// CHECK-LABEL: func.func @_QPtest_copy_in_out_2(
+// CHECK-SAME: %[[ARG_0]]: !fir.box<!fir.array<*:f32>> {fir.bindc_name = "x"}) {
+// CHECK: %[[VAL_0]] = fir.alloca !fir.box<!fir.heap<!fir.array<*:f32>>>
+// CHECK: %[[VAL_1]] = fir.dummy_scope : !fir.dscope
+// CHECK: %[[VAL_2]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %[[VAL_1]] {uniq_name = "_QFtest_copy_in_out_2Ex"} : (!fir.box<!fir.array<*:f32>>, !fir.dscope) -> (!fir.box<!fir.array<*:f32>>, !fir.box<!fir.array<*:f32>>)
+// CHECK: %[[VAL_3]]:2 = hlfir.copy_in %[[VAL_2]]#0 to %[[VAL_0]] : (!fir.box<!fir.array<*:f32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> (!fir.box<!fir.array<*:f32>>, i1)
+// CHECK: fir.call @_QPtakes_contiguous_intentin(%[[VAL_3]]#0) fastmath<contract> : (!fir.box<!fir.array<*:f32>>) -> ()
+// CHECK: hlfir.copy_out %[[VAL_0]], %[[VAL_3]]#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>, i1) -> ()
+// CHECK: return
+// CHECK: }
More information about the flang-commits
mailing list