[flang-commits] [flang] acfbfca - [flang][HLFIR] Relax InlineElementals to support more than two users (#186916)
via flang-commits
flang-commits at lists.llvm.org
Mon Jul 6 07:41:59 PDT 2026
Author: anoopkg6
Date: 2026-07-06T16:41:53+02:00
New Revision: acfbfca988887d830a22fc9dd684ced437928cdb
URL: https://github.com/llvm/llvm-project/commit/acfbfca988887d830a22fc9dd684ced437928cdb
DIFF: https://github.com/llvm/llvm-project/commit/acfbfca988887d830a22fc9dd684ced437928cdb.diff
LOG: [flang][HLFIR] Relax InlineElementals to support more than two users (#186916)
This pr optimizes the lowering of MINLOC and MAXLOC when the MASK
argument is a simple equality comparison (e.g., MASK = A == X).
This patch introduces isEqualityMask to identify these patterns and
inline the comparison directly into the reduction loop.
This allows the SimplifyHLFIRIntrinsicscation pass to extract the
invariant
search target and bypasses the creation of a temporary logical mask
array
by inlining the equality comparison directly into the reduction loop.
optimization removes the 'hlfir.apply' to the mask's hlfir.elemental,
which
gets eliminated in bufferize-hlfir pass.
Simplifies the reduction state by removing the min/max value tracker,
as the target value is already known.
Implements a 'first-hit' locking mechanism.
---------
Co-authored-by: anoop.kumar6 at ibm.com <anoopk at b35lp63.lnxne.boe>
Added:
flang/test/HLFIR/inline-elemental-multi-users.fir
Modified:
flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
Removed:
################################################################################
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
index ff84a3cff0afb..c84f3c373336c 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
@@ -16,9 +16,11 @@
#include "flang/Optimizer/Dialect/Support/FIRContext.h"
#include "flang/Optimizer/HLFIR/HLFIROps.h"
#include "flang/Optimizer/HLFIR/Passes.h"
+#include "mlir/Analysis/AliasAnalysis.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/PatternMatch.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/DialectConversion.h"
@@ -31,16 +33,162 @@ namespace hlfir {
#include "flang/Optimizer/HLFIR/Passes.h.inc"
} // namespace hlfir
-/// If the elemental has only two uses and those two are an apply operation and
-/// a destroy operation, return those two, otherwise return {}
-static std::optional<std::pair<hlfir::ApplyOp, hlfir::DestroyOp>>
-getTwoUses(hlfir::ElementalOp elemental) {
- mlir::Operation::user_range users = elemental->getUsers();
- // don't inline anything with more than one use (plus hfir.destroy)
- if (std::distance(users.begin(), users.end()) != 2) {
- return std::nullopt;
- }
+/// Collects all memory values (buffers/references) that the elemental body
+/// reads from. Use MemoryEffectOpInterface for a fail-safe implementation.
+static mlir::LogicalResult
+getReadDependencies(hlfir::ElementalOp elemental,
+ llvm::SmallVectorImpl<mlir::Value> &deps) {
+ llvm::SmallPtrSet<mlir::Value, 8> seen;
+
+ mlir::WalkResult walkResult =
+ elemental.getRegion().walk([&](mlir::Operation *op) {
+ if (mlir::isMemoryEffectFree(op))
+ return mlir::WalkResult::advance();
+
+ if (auto memInterface =
+ mlir::dyn_cast<mlir::MemoryEffectOpInterface>(op)) {
+ llvm::SmallVector<mlir::MemoryEffects::EffectInstance, 4> effects;
+ memInterface.getEffects(effects);
+ bool hasUnspecifiedRead = false;
+
+ for (const auto &effect : effects) {
+ if (mlir::isa<mlir::MemoryEffects::Read>(effect.getEffect())) {
+ if (mlir::Value val = effect.getValue()) {
+ if (seen.insert(val).second)
+ deps.push_back(val);
+ } else {
+ // Read effect on an unspecified resource (e.g., global state).
+ hasUnspecifiedRead = true;
+ }
+ }
+ }
+
+ // If the op has a read effect but the specific value is unknown,
+ // conservatively capture all potential reference operands.
+ if (hasUnspecifiedRead) {
+ // If there are no operands to track, we can't reason about
+ // the dependency.
+ if (op->getNumOperands() == 0)
+ return mlir::WalkResult::interrupt();
+ for (mlir::Value operand : op->getOperands()) {
+ if (operand.getParentRegion() != &elemental.getRegion()) {
+ if (mlir::isa<fir::ReferenceType, fir::PointerType,
+ fir::HeapType, fir::BoxType>(operand.getType())) {
+ if (seen.insert(operand).second)
+ deps.push_back(operand);
+ }
+ }
+ }
+ }
+ return mlir::WalkResult::advance();
+ }
+
+ // Fail-safe: For operations without the interface, conservatively
+ // assume we cannot reason about the dependency.
+ return mlir::WalkResult::interrupt();
+ });
+
+ return mlir::success(!walkResult.wasInterrupted());
+}
+
+/// Checks if an operation 'op' potentially modifies any memory location that
+/// the elemental reads from (captured in 'deps').
+static bool isConflictingWrite(mlir::Operation *op,
+ const llvm::SmallVectorImpl<mlir::Value> &deps,
+ mlir::AliasAnalysis &aa) {
+ // Use walk to handle nested regions (fir.if, fir.do_loop, etc.) recursively.
+ mlir::WalkResult result = op->walk([&](mlir::Operation *nestedOp) {
+ // Operations explicitly marked as having no memory effects are safe.
+ if (mlir::isMemoryEffectFree(nestedOp))
+ return mlir::WalkResult::advance();
+
+ // Explicitly allow safe HLFIR/FIR metadata/lifetime operations.
+ if (mlir::isa<hlfir::DeclareOp, hlfir::AssociateOp, hlfir::EndAssociateOp,
+ fir::AllocaOp, hlfir::NoReassocOp>(nestedOp))
+ return mlir::WalkResult::advance();
+
+ // Check for explicit memory effects via the interface.
+ if (auto memInterface =
+ mlir::dyn_cast<mlir::MemoryEffectOpInterface>(nestedOp)) {
+ llvm::SmallVector<mlir::MemoryEffects::EffectInstance, 4> effects;
+ memInterface.getEffects(effects);
+
+ for (const auto &effect : effects) {
+ // Analyze effects that modify memory or release resources.
+ if (mlir::isa<mlir::MemoryEffects::Write, mlir::MemoryEffects::Free>(
+ effect.getEffect())) {
+ mlir::Value accessedValue = effect.getValue();
+ // Fail-safe: Assuming conflict for Unknown resource (e.g. external
+ // call).
+ if (!accessedValue)
+ return mlir::WalkResult::interrupt();
+
+ // Perform alias analysis against all read dependencies.
+ for (mlir::Value dep : deps) {
+ if (!aa.alias(accessedValue, dep).isNo())
+ return mlir::WalkResult::interrupt();
+ }
+ }
+ }
+ } else if (nestedOp->getNumRegions() == 0) {
+ // Conservative Fallback: If an operation doesn't have interface and
+ // has no regions (e.g. a fir.call), assume it can modify anything.
+ return mlir::WalkResult::interrupt();
+ }
+
+ return mlir::WalkResult::advance();
+ });
+
+ // Conflict found as walk interrupted.
+ return result.wasInterrupted();
+}
+
+static bool isSafeToInline(hlfir::ElementalOp producer,
+ hlfir::ApplyOp applySite, mlir::AliasAnalysis &aa,
+ mlir::DominanceInfo &domInfo) {
+ if (!domInfo.properlyDominates(producer.getOperation(),
+ applySite.getOperation()))
+ return false;
+ llvm::SmallVector<mlir::Value> deps;
+ if (mlir::failed(getReadDependencies(producer, deps)))
+ return false;
+
+ mlir::Operation *func = producer->getParentOfType<mlir::func::FuncOp>();
+ if (!func)
+ return false;
+
+ // Check for conflicting writes between the producer and the apply site.
+ mlir::WalkResult result = func->walk([&](mlir::Operation *op) {
+ if (op == producer.getOperation() || op == applySite.getOperation())
+ return mlir::WalkResult::advance();
+
+ // Analyze operations in the execution path from producer to applySite.
+ if (domInfo.properlyDominates(producer.getOperation(), op) &&
+ domInfo.dominates(op, applySite.getOperation())) {
+ // If 'op' contains the applySite (like a loop shell), skip it to avoid
+ // false positives. Its internal operations will be visited individually.
+ if (op->getBlock() == applySite.getOperation()->getBlock()) {
+ if (isConflictingWrite(op, deps, aa))
+ return mlir::WalkResult::interrupt();
+ } else if (!op->isAncestor(applySite.getOperation())) {
+ // Check operations in sibling blocks or preceding control-flow paths.
+ if (isConflictingWrite(op, deps, aa))
+ return mlir::WalkResult::interrupt();
+ }
+ }
+ return mlir::WalkResult::advance();
+ });
+
+ return !result.wasInterrupted();
+}
+
+/// Traces the elemental's dataflow to find its unique apply and destroy
+/// operations. Returns the destroy op only if no other consumers require the
+/// array buffer.
+static std::optional<std::pair<hlfir::ApplyOp, hlfir::DestroyOp>>
+getTwoUses(hlfir::ElementalOp elemental, mlir::AliasAnalysis &aliasAnalysis,
+ mlir::DominanceInfo &domInfo) {
// If the ElementalOp must produce a temporary (e.g. for
// finalization purposes), then we cannot inline it.
if (hlfir::elementalOpMustProduceTemp(elemental))
@@ -48,12 +196,130 @@ getTwoUses(hlfir::ElementalOp elemental) {
hlfir::ApplyOp apply;
hlfir::DestroyOp destroy;
- for (mlir::Operation *user : users)
- mlir::TypeSwitch<mlir::Operation *, void>(user)
- .Case([&](hlfir::ApplyOp op) { apply = op; })
- .Case([&](hlfir::DestroyOp op) { destroy = op; });
+ unsigned applyCount = 0;
+ bool hasOtherUsers = false;
+
+ llvm::SmallVector<mlir::Value> worklist;
+ worklist.push_back(elemental.getResult());
+ llvm::SmallPtrSet<mlir::Value, 16> visited;
+ llvm::SmallPtrSet<mlir::Operation *, 4> uniqueApplies;
+
+ while (!worklist.empty()) {
+ mlir::Value current = worklist.pop_back_val();
+ if (!current || !visited.insert(current).second)
+ continue;
- if (!apply || !destroy)
+ for (mlir::OpOperand &use : current.getUses()) {
+ mlir::Operation *user = use.getOwner();
+
+ mlir::TypeSwitch<mlir::Operation *, void>(user)
+ .Case<hlfir::ApplyOp>([&](hlfir::ApplyOp op) {
+ // Use raw operation pointer to ensure each apply site is
+ // counted only once.
+ if (uniqueApplies.insert(op.getOperation()).second) {
+ apply = op;
+ applyCount++;
+ }
+ })
+ .Case<hlfir::DestroyOp>([&](hlfir::DestroyOp op) {
+ // Track the mandatory destroy operation for the elemental expr.
+ destroy = op;
+ })
+ .Case<hlfir::DeclareOp, fir::ConvertOp>([&](mlir::Operation *op) {
+ // Follow the dataflow through all results of the operation.
+ // For hlfir.declare, this catches both the variable and base
+ // results. For fir.convert, this catches the converted result.
+ for (mlir::Value result : op->getResults()) {
+ worklist.push_back(result);
+ }
+ })
+ // Buffer Consumers - These require the destroy to stay.
+ .Case<hlfir::AssociateOp, hlfir::SumOp, hlfir::AssignOp,
+ hlfir::DesignateOp, fir::CallOp, fir::StoreOp, fir::BoxAddrOp>(
+ [&](mlir::Operation *) { hasOtherUsers = true; })
+ .Case<mlir::BranchOpInterface>([&](mlir::BranchOpInterface branch) {
+ for (unsigned i = 0; i < branch->getNumSuccessors(); ++i) {
+ mlir::SuccessorOperands operands = branch.getSuccessorOperands(i);
+ for (unsigned j = 0; j < operands.size(); ++j) {
+ if (operands[j] == current) {
+ // The j-th operand of the branch maps to the j-th block
+ // argument of the successor block.
+ mlir::Block *successor = branch->getSuccessor(i);
+ worklist.push_back(successor->getArgument(j));
+ }
+ }
+ }
+ })
+ .Case<fir::ResultOp>([&](fir::ResultOp op) {
+ mlir::Operation *parent = op->getParentOp();
+ // Only forward if the parent is an op that yields values out.
+ if (parent &&
+ mlir::isa<mlir::RegionBranchOpInterface, fir::IfOp,
+ fir::DoLoopOp, hlfir::ElementalOp>(parent)) {
+ for (auto it : llvm::enumerate(op.getOperands())) {
+ if (it.value() == current) {
+ // Map the result index to the parent's result index.
+ unsigned i = it.index();
+ if (i < parent->getNumResults()) {
+ worklist.push_back(parent->getResult(i));
+ }
+ }
+ }
+ } else {
+ // If it's a terminator for an unknown op.
+ hasOtherUsers = true;
+ }
+ })
+ .Default([&](mlir::Operation *op) {
+ if (op->getNumRegions() > 0) {
+ // Follow the value through metadata ops (declare, convert, etc.)
+ // nested inside regions.
+ op->walk([&](mlir::Operation *innerOp) {
+ for (mlir::Value operand : innerOp->getOperands()) {
+ if (operand == current) {
+ if (auto nestedApply =
+ mlir::dyn_cast<hlfir::ApplyOp>(innerOp)) {
+ // Use a set to prevent double-counting if walker
+ // and worklist hit the same apply site.
+ if (uniqueApplies.insert(nestedApply.getOperation())
+ .second) {
+ apply = nestedApply;
+ applyCount++;
+ }
+ } else if (mlir::isa<hlfir::DeclareOp, fir::ConvertOp>(
+ innerOp)) {
+ // Feed internal metadata results back into the worklist.
+ for (mlir::Value res : innerOp->getResults())
+ worklist.push_back(res);
+ } else if (mlir::isa<hlfir::DestroyOp, fir::ResultOp,
+ mlir::BranchOpInterface>(innerOp)) {
+ // Known safe - control flow and cleanup.
+ } else {
+ // If it's an intrinsic, calls or unknown consumer,
+ // it needs the buffer.
+ hasOtherUsers = true;
+ }
+ }
+ }
+ });
+ } else {
+ // Non-region op not handled by specific Case<> (e.g. hlfir.sum)
+ hasOtherUsers = true;
+ }
+ });
+ if (applyCount > 1)
+ return std::nullopt;
+ }
+ }
+
+ // Only inline if there is a unique 'apply' site. Other users (such as
+ // intrinsic operations) are allowed because scalarizing the elemental
+ // renders the original array result redundant.
+ if (applyCount != 1 || !destroy)
+ return std::nullopt;
+
+ // Verify memory effect and dataflow analysis.
+ if (!isSafeToInline(elemental, apply, aliasAnalysis, domInfo))
return std::nullopt;
// we can't inline if the return type of the yield doesn't match the return
@@ -64,7 +330,9 @@ getTwoUses(hlfir::ElementalOp elemental) {
if (apply.getResult().getType() != yield.getElementValue().getType())
return std::nullopt;
- return std::pair{apply, destroy};
+ // Only return the destroy op if there's exactly one apply and no other users.
+ bool safeToDelete = (applyCount == 1 && !hasOtherUsers);
+ return std::make_pair(apply, safeToDelete ? destroy : nullptr);
}
namespace {
@@ -72,15 +340,19 @@ class InlineElementalConversion
: public mlir::OpRewritePattern<hlfir::ElementalOp> {
public:
using mlir::OpRewritePattern<hlfir::ElementalOp>::OpRewritePattern;
-
+ explicit InlineElementalConversion(mlir::MLIRContext *context,
+ mlir::AliasAnalysis &aa,
+ mlir::DominanceInfo &di)
+ : OpRewritePattern<hlfir::ElementalOp>(context), aliasAnalysis(aa),
+ domInfo(di) {}
llvm::LogicalResult
matchAndRewrite(hlfir::ElementalOp elemental,
mlir::PatternRewriter &rewriter) const override {
std::optional<std::pair<hlfir::ApplyOp, hlfir::DestroyOp>> maybeTuple =
- getTwoUses(elemental);
+ getTwoUses(elemental, aliasAnalysis, domInfo);
if (!maybeTuple)
return rewriter.notifyMatchFailure(
- elemental, "hlfir.elemental does not have two uses");
+ elemental, "hlfir.elemental is not a candidate for inlining");
if (elemental.isOrdered()) {
// We can only inline the ordered elemental into a loop-like
@@ -103,11 +375,21 @@ class InlineElementalConversion
// remove the old elemental and all of the bookkeeping
rewriter.replaceOp(apply, {yield.getElementValue()});
rewriter.eraseOp(yield);
- rewriter.eraseOp(destroy);
- rewriter.eraseOp(elemental);
+ // Only erase the destroy and elemental if the analysis shows it's safe.
+ if (hlfir::DestroyOp destroyOp = maybeTuple->second) {
+ // IR has no users left.
+ if (destroyOp->use_empty())
+ rewriter.eraseOp(destroyOp);
+ if (elemental.getResult().use_empty())
+ rewriter.eraseOp(elemental);
+ }
return mlir::success();
}
+
+private:
+ mlir::AliasAnalysis &aliasAnalysis;
+ mlir::DominanceInfo &domInfo;
};
class InlineElementalsPass
@@ -116,13 +398,16 @@ class InlineElementalsPass
void runOnOperation() override {
mlir::MLIRContext *context = &getContext();
+ // Get AliasAnalysis from the pass manager.
+ mlir::AliasAnalysis &aliasAnalysis = getAnalysis<mlir::AliasAnalysis>();
+ mlir::DominanceInfo &domInfo = getAnalysis<mlir::DominanceInfo>();
mlir::GreedyRewriteConfig config;
// Prevent the pattern driver from merging blocks.
config.setRegionSimplificationLevel(
mlir::GreedySimplifyRegionLevel::Disabled);
mlir::RewritePatternSet patterns(context);
- patterns.insert<InlineElementalConversion>(context);
+ patterns.insert<InlineElementalConversion>(context, aliasAnalysis, domInfo);
if (mlir::failed(mlir::applyPatternsGreedily(
getOperation(), std::move(patterns), config))) {
diff --git a/flang/test/HLFIR/inline-elemental-multi-users.fir b/flang/test/HLFIR/inline-elemental-multi-users.fir
new file mode 100644
index 0000000000000..f5c8b300d6909
--- /dev/null
+++ b/flang/test/HLFIR/inline-elemental-multi-users.fir
@@ -0,0 +1,1120 @@
+// RUN: fir-opt -allow-unregistered-dialect --inline-elementals %s | FileCheck %s
+
+// Test inlining of hlfir.elemental into its hlfir.apply site when the
+// elemental has more than two users.
+
+// Test successful inlining with relaxed inlining.
+func.func @test_safe_loop_inlining(%arg0: !fir.ref<!fir.array<10xf32>>, %i: index) {
+ %c1 = arith.constant 1 : index
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ // Elemental Mask.
+ %elem = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%idx: index):
+ %addr = fir.coordinate_of %arg0, %idx : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ %val = fir.load %addr : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // User 1 - hlfir.apply inside nested loops (The target for inlining).
+ fir.do_loop %arg1 = %c1 to %c10 step %c1 {
+ fir.do_loop %arg2 = %c1 to %c10 step %c1 {
+ %res = hlfir.apply %elem, %arg2 : (!hlfir.expr<10xf32>, index) -> f32
+ %dummy = fir.alloca f32
+ fir.store %res to %dummy : !fir.ref<f32>
+ }
+ }
+
+ // User 2 - Associate (Simulating a shared mask use-case).
+ %temp:3 = hlfir.associate %elem(%shape) : (!hlfir.expr<10xf32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>, i1)
+
+ // User 3 - Destroy
+ hlfir.destroy %elem : !hlfir.expr<10xf32>
+
+ hlfir.end_associate %temp#0, %temp#2 : !fir.ref<!fir.array<10xf32>>, i1
+ return
+}
+// CHECK-LABEL: func.func @test_safe_loop_inlining
+// CHECK: %[[ELEM:.*]] = hlfir.elemental
+// CHECK: fir.do_loop
+// CHECK: fir.do_loop %[[INNER_IDX:.*]] =
+// CHECK: %[[ADDR:.*]] = fir.coordinate_of %arg0, %[[INNER_IDX]]
+// CHECK: %[[VAL:.*]] = fir.load %[[ADDR]]
+// CHECK-NOT: hlfir.apply
+// CHECK: hlfir.associate %[[ELEM]]
+// This verifies that even when an elemental is inlined into an apply site,
+// the destroy operation is preserved because another user (hlfir.associate)
+// still requires the array buffer.
+// CHECK: hlfir.destroy %[[ELEM]]
+
+// Test blocking of incorrect inlining because of alias conflict.
+func.func @test_unsafe_loop_alias_conflict(%arg0: !fir.ref<!fir.array<10xf32>>, %new_val: f32) {
+ %c1 = arith.constant 1 : index
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ // Elemental depends on the values in %arg0 (Producer).
+ %elem = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%idx: index):
+ %addr = fir.coordinate_of %arg0, %idx : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ %val = fir.load %addr : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ fir.do_loop %arg1 = %c1 to %c10 step %c1 {
+ // We modify the array that the elemental needs to read from.
+ // Inlining the elemental here would see the new value.
+ %write_addr = fir.coordinate_of %arg0, %arg1 : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ fir.store %new_val to %write_addr : !fir.ref<f32>
+
+ // Target for inlining.
+ %res = hlfir.apply %elem, %arg1 : (!hlfir.expr<10xf32>, index) -> f32
+
+ %dummy = fir.alloca f32
+ fir.store %res to %dummy : !fir.ref<f32>
+ }
+
+ hlfir.destroy %elem : !hlfir.expr<10xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_unsafe_loop_alias_conflict
+// Elemental should not be inlined, check presence of elemental and apply.
+// CHECK: %[[ELEM:.*]] = hlfir.elemental
+// CHECK: fir.do_loop
+// CHECK: fir.store
+// CHECK: %[[APPLIED:.*]] = hlfir.apply %[[ELEM]]
+// CHECK: fir.store %[[APPLIED]]
+// Inlined code (coordinate_of/load) should not appear inside the loop.
+// CHECK-NOT: fir.coordinate_of %arg0, %arg1
+
+
+// Check successful inlining where 2-d hlfir.elemental survives because the
+// 'associate' op is still using it.
+func.func @test_inlining_use_mask(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: !fir.ref<i32>) {
+ %c1 = arith.constant 1 : index
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10, %c10 : (index, index) -> !fir.shape<2>
+
+ // Elemental Mask.
+ %mask = hlfir.elemental %shape unordered : (!fir.shape<2>) -> !hlfir.expr<10x10x!fir.logical<4>> {
+ ^bb0(%i: index, %j: index):
+ %val = hlfir.designate %arg0 (%i, %j) : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+ %load = fir.load %val : !fir.ref<i32>
+ %ref = fir.load %arg1 : !fir.ref<i32>
+ %cmp = arith.cmpi eq, %load, %ref : i32
+ %res = fir.convert %cmp : (i1) -> !fir.logical<4>
+ hlfir.yield_element %res : !fir.logical<4>
+ }
+
+ // Extra User - This keeps the elemental alive even after inlining the apply
+ // site. Total uses = 3 (associate, apply, destroy).
+ %extra:3 = hlfir.associate %mask : (!hlfir.expr<10x10x!fir.logical<4>>) -> (!fir.box<!fir.array<10x10x!fir.logical<4>>>, !fir.ref<!fir.array<10x10x!fir.logical<4>>>, i1)
+
+ // Apply inside a loop
+ fir.do_loop %arg2 = %c1 to %c10 step %c1 {
+ fir.do_loop %arg3 = %c1 to %c10 step %c1 {
+ %applied = hlfir.apply %mask, %arg3, %arg2 : (!hlfir.expr<10x10x!fir.logical<4>>, index, index) -> !fir.logical<4>
+ %dummy_ref = fir.alloca !fir.logical<4>
+ fir.store %applied to %dummy_ref : !fir.ref<!fir.logical<4>>
+ }
+ }
+
+ hlfir.end_associate %extra#0, %extra#2 : !fir.box<!fir.array<10x10x!fir.logical<4>>>, i1
+ hlfir.destroy %mask : !hlfir.expr<10x10x!fir.logical<4>>
+ return
+}
+// CHECK-LABEL: func.func @test_inlining_use_mask
+// CHECK: %[[MASK:.*]] = hlfir.elemental
+// CHECK: hlfir.associate %[[MASK]]
+// CHECK: fir.do_loop
+// CHECK: fir.do_loop
+// CHECK-NOT: hlfir.apply
+// CHECK: %[[VAL:.*]] = hlfir.designate %arg0
+// CHECK: %[[LOAD:.*]] = fir.load %[[VAL]]
+// CHECK: %[[REF:.*]] = fir.load %arg1
+// CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[LOAD]], %[[REF]]
+// CHECK: %[[RES:.*]] = fir.convert %[[CMP]]
+// CHECK: fir.store %[[RES]]
+// CHECK: hlfir.destroy %[[MASK]]
+
+// Check elemental and destroy are successfully removed even when
+// the expression is passed through metadata operations (fir.convert)
+// and used in nested loops.
+func.func @test_inlining_elemental_cleanup(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: !fir.ref<i32>) {
+ %c1 = arith.constant 1 : index
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10, %c10 : (index, index) -> !fir.shape<2>
+
+ // Elemental Mask.
+ %mask = hlfir.elemental %shape unordered : (!fir.shape<2>) -> !hlfir.expr<10x10x!fir.logical<4>> {
+ ^bb0(%i: index, %j: index):
+ %val = hlfir.designate %arg0 (%i, %j) : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+ %load = fir.load %val : !fir.ref<i32>
+ %ref = fir.load %arg1 : !fir.ref<i32>
+ %cmp = arith.cmpi eq, %load, %ref : i32
+ %res = fir.convert %cmp : (i1) -> !fir.logical<4>
+ hlfir.yield_element %res : !fir.logical<4>
+ }
+
+ %extra = fir.convert %mask : (!hlfir.expr<10x10x!fir.logical<4>>) -> !hlfir.expr<10x10x!fir.logical<4>>
+
+ // Apply Site.
+ fir.do_loop %arg2 = %c1 to %c10 step %c1 {
+ fir.do_loop %arg3 = %c1 to %c10 step %c1 {
+ %applied = hlfir.apply %mask, %arg3, %arg2 : (!hlfir.expr<10x10x!fir.logical<4>>, index, index) -> !fir.logical<4>
+ %dummy_ref = fir.alloca !fir.logical<4>
+ fir.store %applied to %dummy_ref : !fir.ref<!fir.logical<4>>
+ }
+ }
+
+ hlfir.destroy %mask : !hlfir.expr<10x10x!fir.logical<4>>
+ return
+}
+// CHECK-LABEL: func.func @test_inlining_elemental_cleanup
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: fir.convert
+// CHECK: fir.do_loop
+// CHECK: fir.do_loop
+// CHECK-NOT: hlfir.apply
+// CHECK: arith.cmpi eq
+// CHECK-NOT: hlfir.destroy
+
+// Check that inlining is blocked when there is more than one hlfir.apply
+// site for the same elemental.
+func.func @test_multi_apply_no_inlining(%arg0: !hlfir.expr<?xi32>, %target: i32, %shape: !fir.shape<1>) -> (i1, i1) {
+ %c1 = arith.constant 1 : index
+ %c2 = arith.constant 2 : index
+
+ // Producer (Elemental).
+ %mask = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+ ^bb0(%i: index):
+ %val = hlfir.apply %arg0, %i : (!hlfir.expr<?xi32>, index) -> i32
+ %cmp = arith.cmpi eq, %val, %target : i32
+ %log = fir.convert %cmp : (i1) -> !fir.logical<4>
+ hlfir.yield_element %log : !fir.logical<4>
+ }
+
+ // First Apply Site.
+ %apply1 = hlfir.apply %mask, %c1 : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+ %cond1 = fir.convert %apply1 : (!fir.logical<4>) -> i1
+
+ // Second Apply Site.
+ %apply2 = hlfir.apply %mask, %c2 : (!hlfir.expr<?x!fir.logical<4>>, index) -> !fir.logical<4>
+ %cond2 = fir.convert %apply2 : (!fir.logical<4>) -> i1
+
+ // Destroy.
+ hlfir.destroy %mask : !hlfir.expr<?x!fir.logical<4>>
+
+ return %cond1, %cond2 : i1, i1
+}
+// CHECK-LABEL: func.func @test_multi_apply_no_inlining(
+// CHECK-SAME: %[[ARG0:.*]]: !hlfir.expr<?xi32>, %[[TARGET:.*]]: i32, %[[SHAPE:.*]]: !fir.shape<1>)
+// CHECK: %[[MASK:.*]] = hlfir.elemental %[[SHAPE]]
+// CHECK: %[[A1:.*]] = hlfir.apply %[[MASK]], %{{.*}}
+// CHECK: %[[A2:.*]] = hlfir.apply %[[MASK]], %{{.*}}
+// CHECK: hlfir.destroy %[[MASK]]
+
+// Check global store blocks inlining.
+fir.global @mask_storage : !hlfir.expr<10x10x!fir.logical<4>>
+func.func @test_nested_elemental(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: !fir.ref<i32>) {
+ %c1 = arith.constant 1 : index
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10, %c10 : (index, index) -> !fir.shape<2>
+
+ // The Elemental Mask (b * c).
+ %mask = hlfir.elemental %shape unordered : (!fir.shape<2>) -> !hlfir.expr<10x10x!fir.logical<4>> {
+ ^bb0(%i: index, %j: index):
+ %val = hlfir.designate %arg0 (%i, %j) : (!fir.box<!fir.array<?x?xi32>>, index, index) -> !fir.ref<i32>
+ %load = fir.load %val : !fir.ref<i32>
+ %ref = fir.load %arg1 : !fir.ref<i32>
+ %cmp = arith.cmpi eq, %load, %ref : i32
+ %res = fir.convert %cmp : (i1) -> !fir.logical<4>
+ hlfir.yield_element %res : !fir.logical<4>
+ }
+
+ // Total users - 1. fir.store, 2. hlfir.apply, 3. hlfir.destroy.
+ %ptr = fir.address_of(@mask_storage) : !fir.ref<!hlfir.expr<10x10x!fir.logical<4>>>
+ fir.store %mask to %ptr : !fir.ref<!hlfir.expr<10x10x!fir.logical<4>>>
+
+ // Target loop using the mask.
+ fir.do_loop %arg2 = %c1 to %c10 step %c1 {
+ fir.do_loop %arg3 = %c1 to %c10 step %c1 {
+ // CHECK-NOT: hlfir.apply
+ %applied = hlfir.apply %mask, %arg3, %arg2 : (!hlfir.expr<10x10x!fir.logical<4>>, index, index) -> !fir.logical<4>
+ %dummy_ref = fir.alloca !fir.logical<4>
+ fir.store %applied to %dummy_ref : !fir.ref<!fir.logical<4>>
+ }
+ }
+
+ hlfir.destroy %mask : !hlfir.expr<10x10x!fir.logical<4>>
+ return
+}
+// CHECK-LABEL: func.func @test_nested_elemental
+// CHECK: %[[MASK:.*]] = hlfir.elemental
+// CHECK: %[[PTR:.*]] = fir.address_of(@mask_storage)
+// CHECK: fir.store %[[MASK]] to %[[PTR]]
+// Apply site not inlined.
+// CHECK: fir.do_loop
+// CHECK: fir.do_loop
+// CHECK: %[[VAL:.*]] = hlfir.apply %[[MASK]]
+// CHECK: fir.store %[[VAL]]
+// The designate/load should only be inside the elemental.
+// CHECK-NOT: hlfir.designate
+
+// Inlining into a single hlfir.apply (relaxed inlining).
+// a = (b * c)[1]
+func.func @test_scalar_apply_inlining_safe(%b: !fir.ref<!fir.array<10xf32>>, %c1: index) {
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ // Producer(1D Elemental).
+ %prod = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%i: index):
+ %b_addr = fir.coordinate_of %b, %i : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ %b_val = fir.load %b_addr : !fir.ref<f32>
+ %res = arith.addf %b_val, %b_val : f32
+ hlfir.yield_element %res : f32
+ }
+
+ // Scalar Apply (Target for inlining).
+ %scalar_val = hlfir.apply %prod, %c1 : (!hlfir.expr<10xf32>, index) -> f32
+
+ %temp:3 = hlfir.associate %prod(%shape) : (!hlfir.expr<10xf32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>, i1)
+
+ hlfir.destroy %prod : !hlfir.expr<10xf32>
+
+ // Use the result.
+ %dummy = fir.alloca f32
+ fir.store %scalar_val to %dummy : !fir.ref<f32>
+
+ hlfir.end_associate %temp#0, %temp#2 : !fir.ref<!fir.array<10xf32>>, i1
+ return
+}
+
+// CHECK-LABEL: func.func @test_scalar_apply_inlining_safe
+// CHECK: %[[ELEM:.*]] = hlfir.elemental
+// CHECK: hlfir.yield_element
+// CHECK: %[[ADDR:.*]] = fir.coordinate_of %arg0, %arg1
+// CHECK: %[[VAL:.*]] = fir.load %[[ADDR]]
+// CHECK: arith.addf %[[VAL]], %[[VAL]]
+// CHECK-NOT: hlfir.apply
+// CHECK: hlfir.associate %[[ELEM]]
+// CHECK: hlfir.destroy %[[ELEM]]
+
+// Check long chains of elementals.
+// subroutine reproducer(a)
+// real, dimension(:) :: a
+// a = sqrt(a * (a - 1))
+// end subroutine
+func.func @_QPreproducer(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}) {
+ %c0 = arith.constant 0 : index
+ %f1 = arith.constant 1.0 : f32
+ %0:2 = hlfir.declare %arg0 {uniq_name = "_QFreproducerEa"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+ %1:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+ %2 = fir.shape %1#1 : (index) -> !fir.shape<1>
+
+ // tmp1 = a - 1.
+ %tmp1 = hlfir.elemental %2 unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+ ^bb0(%i: index):
+ %a_ref = hlfir.designate %0#0 (%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+ %a_val = fir.load %a_ref : !fir.ref<f32>
+ %sub = arith.subf %a_val, %f1 : f32
+ hlfir.yield_element %sub : f32
+ }
+
+ %dummy = hlfir.no_reassoc %tmp1 : !hlfir.expr<?xf32>
+
+ // tmp2 = a * tmp1.
+ %tmp2 = hlfir.elemental %2 unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+ ^bb0(%j: index):
+ %a_ref_2 = hlfir.designate %0#0 (%j) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+ %t1_val = hlfir.apply %tmp1, %j : (!hlfir.expr<?xf32>, index) -> f32
+ %a_val_2 = fir.load %a_ref_2 : !fir.ref<f32>
+ %mul = arith.mulf %a_val_2, %t1_val : f32
+ hlfir.yield_element %mul : f32
+ }
+
+ // tmp3 = sqrt(tmp2).
+ %tmp3 = hlfir.elemental %2 unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+ ^bb0(%k: index):
+ %t2_val = hlfir.apply %tmp2, %k : (!hlfir.expr<?xf32>, index) -> f32
+ %res = math.sqrt %t2_val : f32
+ hlfir.yield_element %res : f32
+ }
+
+ // Final assignment.
+ hlfir.assign %tmp3 to %0#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+
+ hlfir.destroy %tmp3 : !hlfir.expr<?xf32>
+ hlfir.destroy %tmp2 : !hlfir.expr<?xf32>
+ hlfir.destroy %dummy: !hlfir.expr<?xf32>
+ hlfir.destroy %tmp1 : !hlfir.expr<?xf32>
+ return
+}
+// CHECK-LABEL: func.func @_QPreproducer
+// CHECK: %[[TMP1:.*]] = hlfir.elemental
+// CHECK: hlfir.no_reassoc %[[TMP1]]
+// CHECK-NOT: hlfir.apply
+// CHECK-DAG: arith.subf
+// CHECK-DAG: arith.mulf
+// CHECK-DAG: math.sqrt
+// CHECK: hlfir.assign
+// CHECK: hlfir.destroy %[[TMP1]]
+
+// Check that the ordered elemental is not inlined into another:
+// a = b + c + d (where b + c is ordered)
+func.func private @persistent_user(!hlfir.expr<?xf32>)
+func.func @test_noinline_ordered(%arg0: !hlfir.expr<?xf32>, %arg1: !hlfir.expr<?xf32>, %shape: !fir.shape<1>) {
+ %c1 = arith.constant 1 : index
+
+ // Producer (b + c) - ordered
+ %el_a = hlfir.elemental %shape {ordered = true} : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+ ^bb0(%i: index):
+ %0 = hlfir.apply %arg0, %i : (!hlfir.expr<?xf32>, index) -> f32
+ %1 = hlfir.apply %arg1, %i : (!hlfir.expr<?xf32>, index) -> f32
+ %sum = arith.addf %0, %1 : f32
+ hlfir.yield_element %sum : f32
+ }
+
+ fir.call @persistent_user(%el_a) : (!hlfir.expr<?xf32>) -> ()
+
+ // Consumer (el_a + d)
+ %el_b = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+ ^bb0(%j: index):
+ // This apply must remain because el_a is ordered.
+ %a_val = hlfir.apply %el_a, %j : (!hlfir.expr<?xf32>, index) -> f32
+ %total = arith.addf %a_val, %a_val : f32
+ hlfir.yield_element %total : f32
+ }
+
+ hlfir.destroy %el_a : !hlfir.expr<?xf32>
+ %val = hlfir.apply %el_b, %c1 : (!hlfir.expr<?xf32>, index) -> f32
+ hlfir.destroy %el_b : !hlfir.expr<?xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_noinline_ordered
+// CHECK: %[[PRODUCER:.*]] = hlfir.elemental %{{.*}} {ordered = true}
+// CHECK: fir.call @persistent_user(%[[PRODUCER]])
+// CHECK: %[[CONSUMER:.*]] = hlfir.elemental
+// CHECK: hlfir.apply %[[PRODUCER]], %{{.*}}
+// CHECK: hlfir.destroy %[[PRODUCER]]
+
+// Check that the elemental is not inlined, because its array result
+// must be finalized.
+func.func @test_noinline_due_to_finalization(%arg0: !fir.box<!fir.array<?x!fir.type<_QMtypesTt1{x:f32}>>>, %shape: !fir.shape<1>) {
+ %c1 = arith.constant 1 : index
+
+ // Producer - Derived-type Elemental.
+ %el = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<?x!fir.type<_QMtypesTt1{x:f32}>> {
+ ^bb0(%i: index):
+ %res = fir.alloca !fir.type<_QMtypesTt1{x:f32}>
+ %ld = fir.load %res : !fir.ref<!fir.type<_QMtypesTt1{x:f32}>>
+ hlfir.yield_element %ld : !fir.type<_QMtypesTt1{x:f32}>
+ }
+
+ fir.call @persistent_user(%el) : (!hlfir.expr<?x!fir.type<_QMtypesTt1{x:f32}>>) -> ()
+
+ // This apply must remain because the elemental result requires finalization.
+ %res_ptr = fir.alloca !fir.type<_QMtypesTt1{x:f32}>
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<?x!fir.type<_QMtypesTt1{x:f32}>>, index) -> !hlfir.expr<!fir.type<_QMtypesTt1{x:f32}>>
+ hlfir.assign %apply to %res_ptr : !hlfir.expr<!fir.type<_QMtypesTt1{x:f32}>>, !fir.ref<!fir.type<_QMtypesTt1{x:f32}>>
+
+ // Destroy with 'finalize' keyword, hlfir::elementalOpMustProduceTemp becomes
+ // true.
+ hlfir.destroy %el finalize : !hlfir.expr<?x!fir.type<_QMtypesTt1{x:f32}>>
+ return
+}
+// CHECK-LABEL: func.func @test_noinline_due_to_finalization
+// CHECK: %[[EL:.*]] = hlfir.elemental
+// CHECK: fir.call @persistent_user(%[[EL]])
+// CHECK: %[[APPLY:.*]] = hlfir.apply %[[EL]], %{{.*}}
+// CHECK: hlfir.destroy %[[EL]] finalize
+
+// Test conflicting writes hidden within nested regions (like fir.if) between
+// producer and the apply site.
+func.func @test_nested_region_conflict(%arg0: !fir.ref<f32>, %cond: i1) {
+ %c1 = arith.constant 1 : index
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+
+ // Producer.
+ %elem = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Nested region (fir.if) containing a store.
+ // This tests if your walk(func) correctly sees into the 'then' block.
+ fir.if %cond {
+ %new_val = arith.constant 3.0 : f32
+ fir.store %new_val to %arg0 : !fir.ref<f32>
+ } else {
+ // Distractor op
+ fir.no_reassoc %cond : i1
+ }
+
+ // Apply Site.
+ %res = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%j: index):
+ %val = hlfir.apply %elem, %j : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.yield_element %val : f32
+ }
+
+ hlfir.destroy %elem : !hlfir.expr<1xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_nested_region_conflict
+// CHECK: %[[ELEM:.*]] = hlfir.elemental
+// CHECK: fir.if %{{.*}} {
+// CHECK: fir.store
+// CHECK: }
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply %[[ELEM]]
+// CHECK: hlfir.destroy %[[ELEM]]
+
+// Checks tracking the elemntal result through block arguments to find the
+// hlfir.apply site across a branch, fir.store in the intervening block (^bb1)
+// blocks the inlining. It was getting inlined with relaxed inlining patch.
+func.func @test_cross_block_conflict(%arg0: !fir.ref<f32>, %shape: !fir.shape<1>) {
+ // Producer in Entry Block.
+ %elemental = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Pass value to block argument to maintain dataflow for the worklist.
+ cf.br ^bb1(%elemental : !hlfir.expr<1xf32>)
+
+^bb1(%block_arg: !hlfir.expr<1xf32>):
+ // Conflicting Write.
+ // This store between producer and apply site across blocks.
+ %new_val = arith.constant 2.0 : f32
+ fir.store %new_val to %arg0 : !fir.ref<f32>
+
+ // Apply Site.
+ %res = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%j: index):
+ %val = hlfir.apply %block_arg, %j : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.yield_element %val : f32
+ }
+
+ hlfir.destroy %elemental : !hlfir.expr<1xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_cross_block_conflict
+// CHECK: %[[ELM:.*]] = hlfir.elemental
+// CHECK: cf.br ^bb1(%[[ELM]] : !hlfir.expr<1xf32>)
+// CHECK: ^bb1(%[[BARG:.*]]: !hlfir.expr<1xf32>):
+// CHECK: fir.store {{.*}} to %arg0
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply %[[BARG]]
+// CHECK: hlfir.destroy %[[ELM]]
+
+// External impure procedure that might modify %arg0.
+func.func private @impure_side_effect()
+
+func.func @test_impure_call_conflict(%arg0: !fir.ref<f32>, %shape: !fir.shape<1>) {
+ // Reads from %arg0.
+ %elemental = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Acts as a memory barrier.
+ fir.call @impure_side_effect() : () -> ()
+
+ // Apply Site.
+ %res = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%j: index):
+ %val = hlfir.apply %elemental, %j : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.yield_element %val : f32
+ }
+
+ hlfir.destroy %elemental : !hlfir.expr<1xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_impure_call_conflict
+// CHECK: %[[ELEM:.*]] = hlfir.elemental
+// CHECK: fir.call @impure_side_effect()
+// CHECK: hlfir.apply %[[ELEM]]
+// CHECK: hlfir.destroy %[[ELEM]]
+
+// Check conflicting write to the same memory buffer read by the elemental
+// producer in loop body blocks inlining.
+func.func @test_memory_dependency_with_designate(%arg0: !fir.ref<!fir.array<10xf32>>, %shape: !fir.shape<1>) {
+ %c1 = arith.constant 1 : index
+ %c5 = arith.constant 5 : index
+ %val = arith.constant 2.0 : f32
+
+ // Reads from the whole array %arg0.
+ %elem = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%i: index):
+ %addr = fir.coordinate_of %arg0, %i : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ %load = fir.load %addr : !fir.ref<f32>
+ hlfir.yield_element %load : f32
+ }
+
+ // Modifies one element of the same array, partial write to the same base
+ // buffer.
+ %specific_addr = hlfir.designate %arg0 (%c1) : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ fir.store %val to %specific_addr : !fir.ref<f32>
+
+ // Apply Site.
+ %res = hlfir.elemental %shape unordered : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%j: index):
+ %applied = hlfir.apply %elem, %j : (!hlfir.expr<10xf32>, index) -> f32
+ hlfir.yield_element %applied : f32
+ }
+
+ hlfir.destroy %elem : !hlfir.expr<10xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_memory_dependency_with_designate
+// CHECK: %[[ELEM:.*]] = hlfir.elemental {{.*}} unordered
+// CHECK: fir.store {{.*}} to %{{.*}}
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply %[[ELEM]]
+// CHECK: hlfir.destroy %[[ELEM]]
+
+// Inlining is blocked because an unknown op uses %arg0.
+func.func @test_unknown_op_fallback(%arg0: !fir.ref<f32>) -> f32 {
+ %c1 = arith.constant 1 : index
+ %f1 = arith.constant 1.0 : f32
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ "unknown.op"(%arg0) : (!fir.ref<f32>) -> ()
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ fir.store %f1 to %arg0 : !fir.ref<f32>
+
+ %apply = hlfir.apply %elemental, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %elemental : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_unknown_op_fallback
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
+
+// Inlining is blocked because the walker found a dependency inside a region.
+func.func @test_nested_dependency(%arg0: !fir.ref<f32>, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %f1 = arith.constant 1.0 : f32
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %res = fir.if %cond -> f32 {
+ %val = fir.load %arg0 : !fir.ref<f32>
+ fir.result %val : f32
+ } else {
+ %c0 = arith.constant 0.0 : f32
+ fir.result %c0 : f32
+ }
+ hlfir.yield_element %res : f32
+ }
+
+ fir.store %f1 to %arg0 : !fir.ref<f32>
+
+ %apply = hlfir.apply %elemental, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %elemental : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_nested_dependency
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
+
+// Check if fir.box dependency are captured in fail-safe logic.
+func.func @test_box_dependency(%arg0: !fir.box<!fir.array<?xf32>>) -> f32 {
+ %c1 = arith.constant 1 : index
+ %f0 = arith.constant 0.0 : f32
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ // 'unknown.op' uses the box. The fail-safe should capture %arg0 dependency.
+ "unknown.op"(%arg0) : (!fir.box<!fir.array<?xf32>>) -> ()
+
+ %addr_in = hlfir.designate %arg0 (%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+ %val = fir.load %addr_in : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Inlining blocked as %arg0 was captured.
+ %addr_out = hlfir.designate %arg0 (%c1) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+ fir.store %f0 to %addr_out : !fir.ref<f32>
+
+ %apply = hlfir.apply %elemental, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %elemental : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_box_dependency
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
+
+// Check if .Default walker correctly enters a region (fir.if) and identifies
+// metadata (fir.convert). It then feeds the result back to the worklist,
+// allowing analysis to find the nested hlfir.apply.
+func.func @test_nested_metadata_feedback(%arg0: !fir.shape<1>, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %c0 = arith.constant 5.0 : f32
+ hlfir.yield_element %c0 : f32
+ }
+
+ %res = fir.if %cond -> f32 {
+ %conv1 = fir.convert %el : (!hlfir.expr<1xf32>) -> !hlfir.expr<1xf32>
+ %conv2 = fir.convert %conv1 : (!hlfir.expr<1xf32>) -> !hlfir.expr<1xf32>
+ %apply = hlfir.apply %conv2, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ fir.result %apply : f32
+ } else {
+ %f0 = arith.constant 0.0 : f32
+ fir.result %f0 : f32
+ }
+
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %res : f32
+}
+// CHECK-LABEL: func.func @test_nested_metadata_feedback
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK: %[[VAL:.*]] = arith.constant 5.000000e+00 : f32
+// CHECK: fir.if
+// CHECK: fir.result %{{.*}} : f32
+// CHECK: else
+// CHECK: fir.result %{{.*}} : f32
+
+// Check if analysis can find an hlfir.apply nested inside a fir.if
+// when other operations like hlfir.declare are present in the
+// same region. It ensures the .Default walker doesn't get stuck.
+func.func @test_nested_declare_results(%arg0: !fir.shape<1>, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %cst = arith.constant 1.0 : f32
+ hlfir.yield_element %cst : f32
+ }
+
+ %res = fir.if %cond -> f32 {
+ // Metadata ops present in the region.
+ %mem = fir.alloca !hlfir.expr<1xf32>
+ %v:2 = hlfir.declare %mem {uniq_name = "test_var"} : (!fir.ref<!hlfir.expr<1xf32>>) -> (!fir.ref<!hlfir.expr<1xf32>>, !fir.ref<!hlfir.expr<1xf32>>)
+
+ // The apply using the elemental.
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ fir.result %apply : f32
+ } else {
+ %f0 = arith.constant 0.0 : f32
+ fir.result %f0 : f32
+ }
+
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %res : f32
+}
+// CHECK-LABEL: func.func @test_nested_declare_results
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK: %[[VAL:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK: fir.if
+// CHECK: fir.result %{{.*}} : f32
+// CHECK: else
+// CHECK: fir.result %{{.*}} : f32
+
+// Check fail-safe logic in getReadDependencies.
+// The 'test.unknown_read' operation has no memory interface, so the pass
+// should conservatively capture its operand (%mem) as a dependency.
+// No inlining as there is a fir.store to %mem before the apply.
+func.func @test_unspecified_read_fallback(%mem: !fir.ref<f32>) -> f32 {
+ %c1 = arith.constant 1 : index
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+ %el = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ "test.unknown_read"(%mem) : (!fir.ref<f32>) -> ()
+ %val = fir.load %mem : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+ %f1 = arith.constant 1.0 : f32
+ fir.store %f1 to %mem : !fir.ref<f32>
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_unspecified_read_fallback
+// CHECK: %[[EL:.*]] = hlfir.elemental
+// CHECK: fir.store
+// CHECK: %[[RES:.*]] = hlfir.apply %[[EL]]
+// CHECK: hlfir.destroy %[[EL]]
+// CHECK: return %[[RES]]
+
+// Checks recursive analysis of the .Default walker. It identifies an
+// hlfir.apply nested inside a fir.if block, skipping the container itself
+// and successfully inlining the elemental body into the nested site.
+func.func @test_minimal_nested(%arg0: !fir.shape<1>, %cond: i1) {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %c0 = arith.constant 0.0 : f32
+ hlfir.yield_element %c0 : f32
+ }
+
+ fir.if %cond {
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ "test.sink"(%apply) : (f32) -> ()
+ }
+
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_minimal_nested
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK: arith.constant 0.000000e+00 : f32
+// CHECK: "test.sink"
+
+// Check .Default case identify unknown region-based operations as other users.
+// The pass should inline the hlfir.apply but should preserve the
+// hlfir.elemental and hlfir.destroy because the buffer is still required by
+// unknown op.
+func.func @test_unknown_region_consumer(%arg0: !fir.shape<1>) {
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %c0 = arith.constant 0.0 : f32
+ hlfir.yield_element %c0 : f32
+ }
+
+ // An operation with regions that can't be recognized as a metadata/apply op.
+ "unknown.region_op"(%el) ({
+ ^bb0(%arg1: !hlfir.expr<1xf32>):
+ "some.inner.op"() : () -> ()
+ }) : (!hlfir.expr<1xf32>) -> ()
+
+ %c1 = arith.constant 1 : index
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_unknown_region_consumer
+// CHECK: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK: hlfir.destroy
+
+// Check recursive metadata (Walker -> Worklist -> Walker).
+// hlfir.apply nested within multiple layers of control flow (nested fir.if).
+// .Default walker and main worklist can hand off the elemental result across
+// region boundaries many times.
+func.func @test_deep_hybrid_nesting(%arg0: !fir.shape<1>, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %c0 = arith.constant 42.0 : f32
+ hlfir.yield_element %c0 : f32
+ }
+
+ %res = fir.if %cond -> f32 {
+ // .Default walker finds this, feeds back to worklist
+ %conv = fir.convert %el : (!hlfir.expr<1xf32>) -> !hlfir.expr<1xf32>
+
+ // Worklist finds this nested if, .Default walker runs again.
+ %inner = fir.if %cond -> f32 {
+ %apply = hlfir.apply %conv, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ fir.result %apply : f32
+ } else {
+ %f0 = arith.constant 0.0 : f32
+ fir.result %f0 : f32
+ }
+ fir.result %inner : f32
+ } else {
+ %f0 = arith.constant 0.0 : f32
+ fir.result %f0 : f32
+ }
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %res : f32
+}
+// CHECK-LABEL: func.func @test_deep_hybrid_nesting
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK: arith.constant 4.200000e+01 : f32
+
+// Check conservative fail-safe in getReadDependencies. When an unknown
+// operation is encountered, all of its reference-type operands are captured
+// as dependencies. Inlining is blocked because of any of these captured
+// values.
+func.func @test_multi_operand_fallback(%mem1: !fir.ref<f32>, %mem2: !fir.ref<f32>) -> f32 {
+ %c1 = arith.constant 1 : index
+ %shape = fir.shape %c1 : (index) -> !fir.shape<1>
+
+ %el = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ // Unknown op with two reference operands.
+ "unknown.op"(%mem1, %mem2) : (!fir.ref<f32>, !fir.ref<f32>) -> ()
+ %val = fir.load %mem1 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Write to the second operand (%mem2).
+ // In case of correctly captured all operands of the unknown op,
+ // inlining of the elemental should be blocked due to write to %mem2.
+ %f1 = arith.constant 1.0 : f32
+ fir.store %f1 to %mem2 : !fir.ref<f32>
+
+ %apply = hlfir.apply %el, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_multi_operand_fallback
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
+
+// Check fir::ResultOp mapping logic in the TypeSwitch - analysis can track
+// the elemental result as it is yielded out of a fir.do_loop, correctly maps
+// iter_arg to the loop result to find hlfir.apply site.
+func.func @test_loop_carried_result(%arg0: !fir.shape<1>, %n: index) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %cst = arith.constant 7.0 : f32
+ hlfir.yield_element %cst : f32
+ }
+
+ // The elemental result is passed through a loop result.
+ %loop_res = fir.do_loop %i = %c1 to %n step %c1 iter_args(%arg = %el) -> (!hlfir.expr<1xf32>) {
+ fir.result %arg : !hlfir.expr<1xf32>
+ }
+
+ // The apply uses the result of the loop.
+ // TypeSwitch for fir::ResultOp should map %arg back to %loop_res.
+ %apply = hlfir.apply %loop_res, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_loop_carried_result
+// CHECK-NOT: hlfir.elemental
+// CHECK: arith.constant 7.0
+
+// Check if worklist successfully traverses block boundaries.
+// BranchOpInterface logic correctly maps the elemental result to the
+// successor block's argument, allowing the pass to find hlfir.apply in
+// a separate basic block.
+func.func @test_cross_block_dataflow(%arg0: !fir.shape<1>, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %cst = arith.constant 3.14 : f32
+ hlfir.yield_element %cst : f32
+ }
+
+ // Value is passed as a block argument to bb1.
+ cf.cond_br %cond, ^bb1(%el : !hlfir.expr<1xf32>), ^bb2
+
+^bb1(%block_arg: !hlfir.expr<1xf32>):
+ // The worklist must map %el to %block_arg via BranchOpInterface
+ %apply = hlfir.apply %block_arg, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+
+^bb2:
+ %f0 = arith.constant 0.0 : f32
+ return %f0 : f32
+}
+// CHECK-LABEL: func.func @test_cross_block_dataflow
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK-NOT: hlfir.destroy
+// CHECK: arith.constant 3.14
+
+// This test verifiess following the elemental result through a loop exit.
+// By mapping the fir.result operand to the parent operation's result,
+// the worklist can trace the value from inside the loop to an hlfir.apply
+// site located outside the loop.
+func.func @test_loop_exit_dataflow(%arg0: !fir.shape<1>, %n: index) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %cst = arith.constant 1.23 : f32
+ hlfir.yield_element %cst : f32
+ }
+
+ // The elemental result exits the loop via fir.result.
+ %loop_res = fir.do_loop %i = %c1 to %n step %c1 iter_args(%arg = %el) -> (!hlfir.expr<1xf32>) {
+ fir.result %arg : !hlfir.expr<1xf32>
+ }
+
+ // The apply site is outside the loop, using the loop's result.
+ %apply = hlfir.apply %loop_res, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_loop_exit_dataflow
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK-NOT: hlfir.destroy
+// CHECK: arith.constant 1.23
+
+// Check if worklist can follow a result through nested control flow
+// (if inside a loop) and out of the loop result itself.
+func.func @test_complex_nesting_valid(%arg0: !fir.shape<1>, %n: index, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %cst = arith.constant 100.0 : f32
+ hlfir.yield_element %cst : f32
+ }
+
+ // Value enters the loop via iter_args.
+ %loop = fir.do_loop %i = %c1 to %n step %c1 iter_args(%arg = %el) -> (!hlfir.expr<1xf32>) {
+ // Nested structured control flow.
+ %res = fir.if %cond -> !hlfir.expr<1xf32> {
+ fir.result %arg : !hlfir.expr<1xf32>
+ } else {
+ fir.result %arg : !hlfir.expr<1xf32>
+ }
+ fir.result %res : !hlfir.expr<1xf32>
+ }
+
+ // Value exits the loop and is used by apply.
+ %apply = hlfir.apply %loop, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %apply : f32
+}
+// CHECK-LABEL: func.func @test_complex_nesting_valid
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK-NOT: hlfir.destroy
+// CHECK: arith.constant 1.000000e+02 : f32
+
+// Recursive .Default walker can find an apply site deep within nested regions
+// (if inside a loop).
+func.func @test_deep_nested_discovery(%arg0: !fir.shape<1>, %n: index, %cond: i1) -> f32 {
+ %c1 = arith.constant 1 : index
+ %f0 = arith.constant 0.0 : f32
+ %el = hlfir.elemental %arg0 : (!fir.shape<1>) -> !hlfir.expr<1xf32> {
+ ^bb0(%i: index):
+ %c100 = arith.constant 100.0 : f32
+ hlfir.yield_element %c100 : f32
+ }
+
+ %res = fir.do_loop %i = %c1 to %n step %c1 iter_args(%arg = %el) -> (!hlfir.expr<1xf32>) {
+ %inner = fir.if %cond -> !hlfir.expr<1xf32> {
+ // The .Default walker on the fir.if finds this.
+ %apply = hlfir.apply %arg, %c1 : (!hlfir.expr<1xf32>, index) -> f32
+ "test.sink"(%apply) : (f32) -> ()
+ fir.result %arg : !hlfir.expr<1xf32>
+ } else {
+ fir.result %arg : !hlfir.expr<1xf32>
+ }
+ fir.result %inner : !hlfir.expr<1xf32>
+ }
+
+ hlfir.destroy %el : !hlfir.expr<1xf32>
+ return %f0 : f32
+}
+// CHECK-LABEL: func.func @test_deep_nested_discovery
+// CHECK-NOT: hlfir.elemental
+// CHECK-NOT: hlfir.apply
+// CHECK-NOT: hlfir.destroy
+// CHECK: arith.constant 1.000000e+02 : f32
+
+// Check an elemental is not inlined into a loop if a variable used by the
+// elemental is modified within the loop body before the apply site.
+// This ensures dataflow safety in the execution path.
+func.func @test_loop_body_conflict(%arg0: !fir.ref<f32>, %arg1: !fir.ref<!fir.array<10xf32>>) {
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ // Producer: Elemental depends on the value in %arg0.
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%i: index):
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ // Consumer Loop: Modifies %arg0 before calling hlfir.apply.
+ %c1 = arith.constant 1 : index
+ fir.do_loop %i = %c1 to %c10 step %c1 {
+ // This store should trigger conflict check.
+ %new_val = arith.constant 1.0 : f32
+ fir.store %new_val to %arg0 : !fir.ref<f32>
+
+ // Apply site: Should not be inlined because of the store above.
+ %val_to_store = hlfir.apply %elemental, %i : (!hlfir.expr<10xf32>, index) -> f32
+ %addr = hlfir.designate %arg1 (%i) : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ hlfir.assign %val_to_store to %addr : f32, !fir.ref<f32>
+ }
+
+ hlfir.destroy %elemental : !hlfir.expr<10xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_loop_body_conflict
+// CHECK: hlfir.elemental
+// CHECK: fir.do_loop
+// CHECK: fir.store
+// CHECK: hlfir.apply
+// CHECK: hlfir.destroy
+
+// Check sibling block conflict: apply is in then block, but the store is in
+// the else block where arg0 is modified. Inlining is not safe.
+func.func @test_sibling_block_conflict(%arg0: !fir.ref<f32>, %arg1: !fir.ref<!fir.array<10xf32>>, %cond: i1) {
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ // Producer depends on %arg0.
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%i: index):
+ %val = fir.load %arg0 : !fir.ref<f32>
+ hlfir.yield_element %val : f32
+ }
+
+ %c1 = arith.constant 1 : index
+ fir.do_loop %i = %c1 to %c10 step %c1 {
+ fir.if %cond {
+ // Apply site.
+ %val_to_store = hlfir.apply %elemental, %i : (!hlfir.expr<10xf32>, index) -> f32
+ %addr = hlfir.designate %arg1 (%i) : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ hlfir.assign %val_to_store to %addr : f32, !fir.ref<f32>
+ } else {
+ // Conflict: Sibling block modifies the dependency.
+ %new_val = arith.constant 2.0 : f32
+ fir.store %new_val to %arg0 : !fir.ref<f32>
+ }
+ }
+
+ hlfir.destroy %elemental : !hlfir.expr<10xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_sibling_block_conflict
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
+
+// Declare an external function with side effects.
+func.func private @opaque_call() -> f32
+// Check if inlining is blocked for fir.call inside the elemental block.
+func.func @test_opaque_call_conflict(%arg1: !fir.ref<!fir.array<10xf32>>) {
+ %c10 = arith.constant 10 : index
+ %shape = fir.shape %c10 : (index) -> !fir.shape<1>
+
+ %elemental = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<10xf32> {
+ ^bb0(%i: index):
+ %val = fir.call @opaque_call() : () -> f32
+ hlfir.yield_element %val : f32
+ }
+
+ %c1 = arith.constant 1 : index
+ fir.do_loop %i = %c1 to %c10 step %c1 {
+ %val_to_store = hlfir.apply %elemental, %i : (!hlfir.expr<10xf32>, index) -> f32
+ %addr = hlfir.designate %arg1 (%i) : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+ hlfir.assign %val_to_store to %addr : f32, !fir.ref<f32>
+ }
+
+ hlfir.destroy %elemental : !hlfir.expr<10xf32>
+ return
+}
+// CHECK-LABEL: func.func @test_opaque_call_conflict
+// CHECK: hlfir.elemental
+// CHECK: hlfir.apply
More information about the flang-commits
mailing list