[Mlir-commits] [mlir] [MLIR][Affine] Handle aliases and effects in fusion dependencies (PR #213413)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Wed Aug 5 03:58:34 PDT 2026
https://github.com/1sgtpepper updated https://github.com/llvm/llvm-project/pull/213413
>From a0b83c0beaacdf8b9822d225c8c6f91c43905b3e Mon Sep 17 00:00:00 2001
From: 1sgtpepper <cynejarviszarceno at gmail.com>
Date: Wed, 5 Aug 2026 18:42:35 +0800
Subject: [PATCH] [MLIR][Affine] Handle aliases and effects in fusion
dependencies
Prevent affine-loop-fusion from reordering memory effects it cannot represent or schedule safely. Keep storage and exact-SSA dependence identity distinct, reject unmodelled effects before schedule changes, and treat failed affine dependence analysis as a possible dependence.
Fixes #211599. The failed-dependence predicates follow the fail-closed analysis work in PR #211014.
---
.../Dialect/Affine/Analysis/AffineAnalysis.h | 16 +-
.../mlir/Dialect/Affine/Analysis/Utils.h | 62 +-
.../mlir/Dialect/Affine/LoopFusionUtils.h | 8 +-
.../Affine/Analysis/AffineAnalysis.cpp | 7 +-
.../Dialect/Affine/Analysis/CMakeLists.txt | 1 +
.../Dialect/Affine/Analysis/LoopAnalysis.cpp | 5 +-
mlir/lib/Dialect/Affine/Analysis/Utils.cpp | 468 ++++++----
.../Dialect/Affine/Transforms/LoopFusion.cpp | 105 ++-
mlir/lib/Dialect/Affine/Utils/CMakeLists.txt | 1 +
.../Dialect/Affine/Utils/LoopFusionUtils.cpp | 146 ++-
mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp | 59 +-
mlir/lib/Dialect/Affine/Utils/Utils.cpp | 2 +-
mlir/test/Dialect/Affine/loop-fusion-2.mlir | 24 +-
mlir/test/Dialect/Affine/loop-fusion-3.mlir | 15 +-
mlir/test/Dialect/Affine/loop-fusion-4.mlir | 866 +++++++++++++++++-
.../Affine/loop-fusion-dependence-check.mlir | 60 +-
.../Affine/loop-fusion-edge-kinds.mlir | 11 +
.../Dialect/Affine/loop-fusion-failure.mlir | 30 +
.../Dialect/Affine/loop-fusion-utilities.mlir | 17 +-
mlir/test/Dialect/Affine/loop-fusion.mlir | 91 +-
.../Analysis/TestMemRefDependenceCheck.cpp | 2 +-
.../lib/Dialect/Affine/TestLoopFusion.cpp | 113 +++
22 files changed, 1818 insertions(+), 291 deletions(-)
create mode 100644 mlir/test/Dialect/Affine/loop-fusion-edge-kinds.mlir
create mode 100644 mlir/test/Dialect/Affine/loop-fusion-failure.mlir
diff --git a/mlir/include/mlir/Dialect/Affine/Analysis/AffineAnalysis.h b/mlir/include/mlir/Dialect/Affine/Analysis/AffineAnalysis.h
index 3e4b8648061ff..a001be9c42346 100644
--- a/mlir/include/mlir/Dialect/Affine/Analysis/AffineAnalysis.h
+++ b/mlir/include/mlir/Dialect/Affine/Analysis/AffineAnalysis.h
@@ -176,12 +176,18 @@ DependenceResult checkMemrefAccessDependence(
SmallVector<DependenceComponent, 2> *dependenceComponents = nullptr,
bool allowRAR = false);
-/// Utility function that returns true if the provided DependenceResult
-/// corresponds to a dependence result.
-inline bool hasDependence(DependenceResult result) {
+/// Returns true if the provided DependenceResult proves that a dependence
+/// exists.
+inline bool mustHaveDependence(DependenceResult result) {
return result.value == DependenceResult::HasDependence;
}
+/// Returns true unless the provided DependenceResult proves that no dependence
+/// exists.
+inline bool mayHaveDependence(DependenceResult result) {
+ return result.value != DependenceResult::NoDependence;
+}
+
/// Returns true if the provided DependenceResult corresponds to the absence of
/// a dependence.
inline bool noDependence(DependenceResult result) {
@@ -190,8 +196,8 @@ inline bool noDependence(DependenceResult result) {
/// Returns in 'depCompsVec', dependence components for dependences between all
/// load and store ops in loop nest rooted at 'forOp', at loop depths in range
-/// [1, maxLoopDepth].
-void getDependenceComponents(
+/// [1, maxLoopDepth]. Returns failure if any dependence cannot be analyzed.
+LogicalResult getDependenceComponents(
AffineForOp forOp, unsigned maxLoopDepth,
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec);
diff --git a/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h b/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
index df4145db90a61..f9347a122be12 100644
--- a/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Analysis/Utils.h
@@ -32,9 +32,8 @@ class AffineForOp;
class AffineValueMap;
struct MemRefAccess;
-// LoopNestStateCollector walks loop nests and collects load and store
-// operations, and whether or not a region holding op other than ForOp and IfOp
-// was encountered in the loop nest.
+// LoopNestStateCollector walks loop nests and collects affine and non-affine
+// memory operations.
struct LoopNestStateCollector {
SmallVector<AffineForOp, 4> forOps;
// Affine loads.
@@ -47,9 +46,11 @@ struct LoopNestStateCollector {
SmallVector<Operation *, 4> memrefStores;
// Free operations.
SmallVector<Operation *, 4> memrefFrees;
+ // True when an operation has effects that cannot be represented by the
+ // memref-keyed dependence graph.
+ bool hasUnmodeledMemoryEffects = false;
- // Collects load and store operations, and whether or not a region holding op
- // other than ForOp and IfOp was encountered in the loop nest.
+ // Collects affine and non-affine memory operations in the loop nest.
void collect(Operation *opToWalk);
};
@@ -127,13 +128,13 @@ struct MemRefDependenceGraph {
// If this edge is stored in Edge = Node.outEdges[i], then
// 'Node.outEdges[i].id' is the identifier of the dest node of the edge.
unsigned id;
- // The SSA value on which this edge represents a dependence.
- // If the value is a memref, then the dependence is between graph nodes
- // which contain accesses to the same memref 'value'. If the value is a
- // non-memref value, then the dependence is between a graph node which
- // defines an SSA value and another graph node which uses the SSA value
- // (e.g. a constant or load operation defining a value which is used inside
- // a loop nest).
+ // Whether this is a storage-level memory dependence or an exact SSA
+ // dependence.
+ enum class Kind { Memory, SSA };
+ Kind kind;
+ // The value on which this edge represents a dependence. Memory edges carry
+ // the canonical representative of a view-like storage class. SSA edges
+ // retain the exact defining value, including for memrefs.
Value value;
};
@@ -158,8 +159,9 @@ struct MemRefDependenceGraph {
// side-effect-free operations with zero results and no regions. Assigns each
// node in the graph a node id based on the order in block. Fails if certain
// kinds of operations, for which `Node` creation isn't supported, are
- // encountered (unknown region holding ops). If `fullAffineDependences` is
- // set, affine memory dependence analysis is performed before concluding that
+ // encountered (unknown effects or region holding ops). If
+ // `fullAffineDependences` is set, affine memory dependence analysis is
+ // performed before concluding that
// conflicting affine memory accesses lead to a dependence check; otherwise, a
// pair of conflicting affine memory accesses (where one of them is a store
// and they are to the same memref) always leads to an edge (conservatively).
@@ -192,16 +194,20 @@ struct MemRefDependenceGraph {
// argument to) the block. Returns false otherwise.
bool writesToLiveInOrEscapingMemrefs(unsigned id) const;
- // Returns true iff there is an edge from node 'srcId' to node 'dstId' which
- // is for 'value' if non-null, or for any value otherwise. Returns false
- // otherwise.
+ // Returns true iff there is any edge from node 'srcId' to node 'dstId'
+ // which is for 'value' if non-null, or for any value otherwise.
bool hasEdge(unsigned srcId, unsigned dstId, Value value = nullptr) const;
- // Adds an edge from node 'srcId' to node 'dstId' for 'value'.
- void addEdge(unsigned srcId, unsigned dstId, Value value);
+ // Returns true iff there is an edge of 'kind' from node 'srcId' to node
+ // 'dstId' which is for 'value' if non-null, or for any value otherwise.
+ bool hasEdge(unsigned srcId, unsigned dstId, Value value,
+ Edge::Kind kind) const;
- // Removes an edge from node 'srcId' to node 'dstId' for 'value'.
- void removeEdge(unsigned srcId, unsigned dstId, Value value);
+ // Adds an edge of 'kind' from node 'srcId' to node 'dstId' for 'value'.
+ void addEdge(unsigned srcId, unsigned dstId, Value value, Edge::Kind kind);
+
+ // Removes an edge of 'kind' from node 'srcId' to node 'dstId' for 'value'.
+ void removeEdge(unsigned srcId, unsigned dstId, Value value, Edge::Kind kind);
// Returns true if there is a path in the dependence graph from node 'srcId'
// to node 'dstId'. Returns false otherwise. `srcId`, `dstId`, and the
@@ -245,20 +251,20 @@ struct MemRefDependenceGraph {
ArrayRef<Operation *> memrefStores,
ArrayRef<Operation *> memrefFrees);
- void clearNodeLoadAndStores(unsigned id);
+ void clearNodeMemoryOps(unsigned id);
- // Calls 'callback' for each input edge incident to node 'id' which carries a
- // memref dependence.
+ // Calls 'callback' for each input edge incident to node 'id' whose value is a
+ // memref, whether it carries a memory or an SSA dependence.
void forEachMemRefInputEdge(unsigned id,
const std::function<void(Edge)> &callback);
- // Calls 'callback' for each output edge from node 'id' which carries a
- // memref dependence.
+ // Calls 'callback' for each output edge from node 'id' whose value is a
+ // memref, whether it carries a memory or an SSA dependence.
void forEachMemRefOutputEdge(unsigned id,
const std::function<void(Edge)> &callback);
- // Calls 'callback' for each edge in 'edges' which carries a memref
- // dependence.
+ // Calls 'callback' for each edge in 'edges' whose value is a memref, whether
+ // it carries a memory or an SSA dependence.
void forEachMemRefEdge(ArrayRef<Edge> edges,
const std::function<void(Edge)> &callback);
diff --git a/mlir/include/mlir/Dialect/Affine/LoopFusionUtils.h b/mlir/include/mlir/Dialect/Affine/LoopFusionUtils.h
index 0ef39fd7d1463..01926897fb7c5 100644
--- a/mlir/include/mlir/Dialect/Affine/LoopFusionUtils.h
+++ b/mlir/include/mlir/Dialect/Affine/LoopFusionUtils.h
@@ -104,8 +104,12 @@ class FusionStrategy {
/// 'Success' if fusion of the src/dst loop nests is feasible (i.e. they are
/// in the same block and dependences would not be violated). Otherwise
/// returns a FusionResult explaining why fusion is not feasible.
-/// NOTE: This function is not feature complete and should only be used in
-/// testing.
+/// NOTE: This utility is not a complete legality check. In particular, it
+/// does not model generic or unknown memory effects in operations between the
+/// candidate loops. Callers requiring complete legality must first perform the
+/// corresponding MemRefDependenceGraph checks; the affine loop-fusion pass
+/// calls this utility only after those checks, and other direct callers should
+/// use it only for testing.
FusionResult
canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp, unsigned dstLoopDepth,
ComputationSliceState *srcSlice,
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
index 3d1a73417d1ea..f86c5e0e8421e 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
@@ -694,7 +694,7 @@ DependenceResult mlir::affine::checkMemrefAccessDependence(
/// Gathers dependence components for dependences between all ops in loop nest
/// rooted at 'forOp' at loop depths in range [1, maxLoopDepth].
-void mlir::affine::getDependenceComponents(
+LogicalResult mlir::affine::getDependenceComponents(
AffineForOp forOp, unsigned maxLoopDepth,
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec) {
// Collect all load and store ops in loop nest rooted at 'forOp'.
@@ -719,9 +719,12 @@ void mlir::affine::getDependenceComponents(
DependenceResult result = checkMemrefAccessDependence(
srcAccess, dstAccess, d, /*dependenceConstraints=*/nullptr,
&depComps);
- if (hasDependence(result))
+ if (result.value == DependenceResult::Failure)
+ return failure();
+ if (mustHaveDependence(result))
depCompsVec->push_back(depComps);
}
}
}
+ return success();
}
diff --git a/mlir/lib/Dialect/Affine/Analysis/CMakeLists.txt b/mlir/lib/Dialect/Affine/Analysis/CMakeLists.txt
index 3a1996349dbed..6c52f7751f1f0 100644
--- a/mlir/lib/Dialect/Affine/Analysis/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Analysis/CMakeLists.txt
@@ -18,6 +18,7 @@ add_mlir_dialect_library(MLIRAffineAnalysis
MLIRControlFlowInterfaces
MLIRDialectUtils
MLIRInferTypeOpInterface
+ MLIRMemRefUtils
MLIRSideEffectInterfaces
MLIRPresburger
MLIRSCFDialect
diff --git a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
index 40802cc6e85e5..2dbc1baa02236 100644
--- a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
@@ -524,8 +524,11 @@ bool mlir::affine::isTilingValid(ArrayRef<AffineForOp> loops) {
srcAccess, dstAccess, d, /*dependenceConstraints=*/nullptr,
&depComps);
+ if (result.value == DependenceResult::Failure)
+ return false;
+
// Skip if there is no dependence in this case.
- if (!hasDependence(result))
+ if (noDependence(result))
continue;
// Check whether there is any negative direction vector in the
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index 321c8e34d907c..1e8de3ffddefd 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -19,6 +19,7 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/IR/IntegerSet.h"
#include "llvm/ADT/SetVector.h"
@@ -38,33 +39,167 @@ using llvm::SmallDenseMap;
using Node = MemRefDependenceGraph::Node;
-// LoopNestStateCollector walks loop nests and collects load and store
-// operations, and whether or not a region holding op other than ForOp and IfOp
-// was encountered in the loop nest.
+static Value canonicalizeMemref(Value value) {
+ if (!value || !isa<BaseMemRefType>(value.getType()))
+ return value;
+ // Use the storage source for graph identity. Keep the original view in
+ // MemRefAccess so affine maps remain expressed in their original
+ // coordinate systems.
+ return memref::skipViewLikeOps(cast<MemrefValue>(value));
+}
+
+static bool isSameMemref(Value lhs, Value rhs) {
+ return canonicalizeMemref(lhs) == canonicalizeMemref(rhs);
+}
+
+static bool edgeMatchesMemref(const MemRefDependenceGraph::Edge &edge,
+ Value memref) {
+ if (edge.kind == MemRefDependenceGraph::Edge::Kind::Memory)
+ return isSameMemref(edge.value, memref);
+ return edge.value == memref;
+}
+
+/// Returns true if an affine memory interface operation has an effect outside
+/// the access represented by its affine interface. The affine access is the
+/// graph's precise model; any additional effect must be represented
+/// separately or the graph must fail closed.
+static bool hasUnmodeledAffineMemoryEffects(Operation *op) {
+ auto affineRead = dyn_cast<AffineReadOpInterface>(op);
+ auto affineWrite = dyn_cast<AffineWriteOpInterface>(op);
+ if (!affineRead && !affineWrite)
+ return false;
+
+ // An operation with both roles has no single affine access model here.
+ if (affineRead && affineWrite)
+ return true;
+
+ auto memInterface = dyn_cast<MemoryEffectOpInterface>(op);
+ if (!memInterface) {
+ // The affine access interface does not establish that the operation has
+ // no additional memory effects.
+ return true;
+ }
+
+ SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4> effects;
+ memInterface.getEffects(effects);
+ if (effects.size() != 1)
+ return true;
+
+ const auto &effect = effects.front();
+ if (effect.getResource() != SideEffects::DefaultResource::get() ||
+ effect.getEffectOnFullRegion())
+ return true;
+ Value memref = affineRead ? affineRead.getMemRef() : affineWrite.getMemRef();
+ if (effect.getValue() != memref)
+ return true;
+ if (affineRead && !isa<MemoryEffects::Read>(effect.getEffect()))
+ return true;
+ if (affineWrite && !isa<MemoryEffects::Write>(effect.getEffect()))
+ return true;
+ return false;
+}
+
+/// Returns true if an effect can be represented by the MDG's per-memref model.
+/// Allocation is represented only for a memref result defined by the same
+/// operation; candidate-loop legality rejects all allocation effects because
+/// slicing can otherwise change their dynamic execution count.
+static bool isRepresentableMemoryEffect(
+ Operation *op,
+ const SideEffects::EffectInstance<MemoryEffects::Effect> &effect) {
+ if (isa<MemoryEffects::Read, MemoryEffects::Write, MemoryEffects::Free>(
+ effect.getEffect()))
+ return effect.getValue() &&
+ effect.getResource() == SideEffects::DefaultResource::get() &&
+ isa<BaseMemRefType>(effect.getValue().getType());
+
+ if (isa<MemoryEffects::Allocate>(effect.getEffect())) {
+ Value value = effect.getValue();
+ // Allocation effects on a result are represented by the exact SSA edge
+ // from the defining operation. Standard memref allocations use both the
+ // default and automatic-allocation resources, so the result identity is
+ // the relevant representation rather than the resource or
+ // effectOnFullRegion.
+ return value && isa<BaseMemRefType>(value.getType()) &&
+ value.getDefiningOp() == op;
+ }
+
+ return false;
+}
+
+/// Returns the values that `op` may have a memref effect of type `EffectTys`
+/// on, not considering recursive effects. View-like values are canonicalized
+/// to their storage source so the MDG uses one key for a view chain while raw
+/// views remain available for affine-coordinate analysis. Unknown effects
+/// cannot be represented by the memref-keyed graph.
+/// Returns false if a selected effect cannot be represented by a memref value
+/// or if the operation's effects are unknown. The MDG has no resource-level or
+/// all-memory edges, so dropping such an effect would make fusion unsound.
+template <typename... EffectTys>
+static bool getMayAffectedValues(Operation *op,
+ SmallVectorImpl<Value> &values) {
+ auto memOp = dyn_cast<MemoryEffectOpInterface>(op);
+ if (!memOp)
+ return !hasUnknownEffects(op);
+ SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4> effects;
+ memOp.getEffects(effects);
+ for (auto &effect : effects) {
+ if (!isa<EffectTys...>(effect.getEffect()))
+ continue;
+ Value effectVal = effect.getValue();
+ if (!effectVal || !isa<BaseMemRefType>(effectVal.getType()))
+ return false;
+ values.push_back(canonicalizeMemref(effectVal));
+ };
+ return true;
+}
+
+/// Returns true if `op` may have a memory effect of type `EffectTys` on
+/// `memref`, i.e., whether `memref` is among the values returned by
+/// `getMayAffectedValues` for `op`. An unrepresentable effect is
+/// conservatively treated as affecting every memref.
+template <typename... EffectTys>
+static bool mayHaveEffect(Operation *op, Value memref) {
+ SmallVector<Value> values;
+ if (!getMayAffectedValues<EffectTys...>(op, values))
+ return true;
+ return llvm::is_contained(values, canonicalizeMemref(memref));
+}
+
+// LoopNestStateCollector walks loop nests and collects affine and non-affine
+// memory operations.
void LoopNestStateCollector::collect(Operation *opToWalk) {
opToWalk->walk([&](Operation *op) {
if (auto forOp = dyn_cast<AffineForOp>(op)) {
forOps.push_back(forOp);
} else if (isa<AffineReadOpInterface>(op)) {
+ hasUnmodeledMemoryEffects |= hasUnmodeledAffineMemoryEffects(op);
loadOpInsts.push_back(op);
} else if (isa<AffineWriteOpInterface>(op)) {
+ hasUnmodeledMemoryEffects |= hasUnmodeledAffineMemoryEffects(op);
storeOpInsts.push_back(op);
} else {
auto memInterface = dyn_cast<MemoryEffectOpInterface>(op);
if (!memInterface) {
- if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
- // This op itself is memory-effect free.
+ if (hasUnknownEffects(op))
+ hasUnmodeledMemoryEffects = true;
+ SmallVector<Value> affectedValues;
+ if (getMayAffectedValues<MemoryEffects::Read>(op, affectedValues) &&
+ affectedValues.empty())
return;
- // Check operands. Eg. ops like the `call` op are handled here.
- for (Value v : op->getOperands()) {
- if (!isa<MemRefType>(v.getType()))
- continue;
- // Conservatively, we assume the memref is read and written to.
- memrefLoads.push_back(op);
- memrefStores.push_back(op);
- }
+ // Unknown effects cannot be represented by the memref-keyed graph.
+ memrefLoads.push_back(op);
+ memrefStores.push_back(op);
} else {
- // Non-affine loads and stores.
+ SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4>
+ effects;
+ memInterface.getEffects(effects);
+ for (const auto &effect : effects)
+ if (!isRepresentableMemoryEffect(op, effect))
+ hasUnmodeledMemoryEffects = true;
+
+ // Non-affine loads, stores, and frees. Allocation effects on local
+ // memref results remain represented by SSA edges; other allocation or
+ // resource effects make the graph fail closed.
if (hasEffect<MemoryEffects::Read>(op))
memrefLoads.push_back(op);
if (hasEffect<MemoryEffects::Write>(op))
@@ -80,12 +215,9 @@ unsigned Node::getLoadOpCount(Value memref) const {
unsigned loadOpCount = 0;
for (Operation *loadOp : loads) {
// Common case: affine reads.
- if (auto affineLoad = dyn_cast<AffineReadOpInterface>(loadOp)) {
- if (memref == affineLoad.getMemRef())
+ if (auto affineLoad = dyn_cast<AffineReadOpInterface>(loadOp))
+ if (isSameMemref(memref, affineLoad.getMemRef()))
++loadOpCount;
- } else if (hasEffect<MemoryEffects::Read>(loadOp, memref)) {
- ++loadOpCount;
- }
}
return loadOpCount;
}
@@ -96,10 +228,9 @@ unsigned Node::getStoreOpCount(Value memref) const {
for (auto *storeOp : llvm::concat<Operation *const>(stores, memrefStores)) {
// Common case: affine writes.
if (auto affineStore = dyn_cast<AffineWriteOpInterface>(storeOp)) {
- if (memref == affineStore.getMemRef())
+ if (isSameMemref(memref, affineStore.getMemRef()))
++storeOpCount;
- } else if (hasEffect<MemoryEffects::Write>(const_cast<Operation *>(storeOp),
- memref)) {
+ } else if (mayHaveEffect<MemoryEffects::Write>(storeOp, memref)) {
++storeOpCount;
}
}
@@ -112,9 +243,9 @@ unsigned Node::hasStore(Value memref) const {
llvm::concat<Operation *const>(stores, memrefStores),
[&](Operation *storeOp) {
if (auto affineStore = dyn_cast<AffineWriteOpInterface>(storeOp)) {
- if (memref == affineStore.getMemRef())
+ if (isSameMemref(memref, affineStore.getMemRef()))
return true;
- } else if (hasEffect<MemoryEffects::Write>(storeOp, memref)) {
+ } else if (mayHaveEffect<MemoryEffects::Write>(storeOp, memref)) {
return true;
}
return false;
@@ -123,7 +254,7 @@ unsigned Node::hasStore(Value memref) const {
unsigned Node::hasFree(Value memref) const {
return llvm::any_of(memrefFrees, [&](Operation *freeOp) {
- return hasEffect<MemoryEffects::Free>(freeOp, memref);
+ return mayHaveEffect<MemoryEffects::Free>(freeOp, memref);
});
}
@@ -131,7 +262,7 @@ unsigned Node::hasFree(Value memref) const {
void Node::getStoreOpsForMemref(Value memref,
SmallVectorImpl<Operation *> *storeOps) const {
for (Operation *storeOp : stores) {
- if (memref == cast<AffineWriteOpInterface>(storeOp).getMemRef())
+ if (isSameMemref(memref, cast<AffineWriteOpInterface>(storeOp).getMemRef()))
storeOps->push_back(storeOp);
}
}
@@ -140,7 +271,7 @@ void Node::getStoreOpsForMemref(Value memref,
void Node::getLoadOpsForMemref(Value memref,
SmallVectorImpl<Operation *> *loadOps) const {
for (Operation *loadOp : loads) {
- if (memref == cast<AffineReadOpInterface>(loadOp).getMemRef())
+ if (isSameMemref(memref, cast<AffineReadOpInterface>(loadOp).getMemRef()))
loadOps->push_back(loadOp);
}
}
@@ -151,41 +282,17 @@ void Node::getLoadAndStoreMemrefSet(
DenseSet<Value> *loadAndStoreMemrefSet) const {
llvm::SmallDenseSet<Value, 2> loadMemrefs;
for (Operation *loadOp : loads) {
- loadMemrefs.insert(cast<AffineReadOpInterface>(loadOp).getMemRef());
+ loadMemrefs.insert(
+ canonicalizeMemref(cast<AffineReadOpInterface>(loadOp).getMemRef()));
}
for (Operation *storeOp : stores) {
- auto memref = cast<AffineWriteOpInterface>(storeOp).getMemRef();
+ auto memref =
+ canonicalizeMemref(cast<AffineWriteOpInterface>(storeOp).getMemRef());
if (loadMemrefs.count(memref) > 0)
loadAndStoreMemrefSet->insert(memref);
}
}
-/// Returns the values that this op has a memref effect of type `EffectTys` on,
-/// not considering recursive effects.
-template <typename... EffectTys>
-static void getEffectedValues(Operation *op, SmallVectorImpl<Value> &values) {
- auto memOp = dyn_cast<MemoryEffectOpInterface>(op);
- if (!memOp) {
- if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
- // No effects.
- return;
- // Memref operands have to be considered as being affected.
- for (Value operand : op->getOperands()) {
- if (isa<MemRefType>(operand.getType()))
- values.push_back(operand);
- }
- return;
- }
- SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4> effects;
- memOp.getEffects(effects);
- for (auto &effect : effects) {
- Value effectVal = effect.getValue();
- if (isa<EffectTys...>(effect.getEffect()) && effectVal &&
- isa<MemRefType>(effectVal.getType()))
- values.push_back(effectVal);
- };
-}
-
/// Add `op` to MDG creating a new node and adding its memory accesses (affine
/// or non-affine to memrefAccesses (memref -> list of nodes with accesses) map.
static Node *
@@ -196,46 +303,43 @@ addNodeToMDG(Operation *nodeOp, MemRefDependenceGraph &mdg,
// all loads and store accesses it contains.
LoopNestStateCollector collector;
collector.collect(nodeOp);
+ if (collector.hasUnmodeledMemoryEffects)
+ return nullptr;
unsigned newNodeId = mdg.nextNodeId++;
Node &node = nodes.insert({newNodeId, Node(newNodeId, nodeOp)}).first->second;
for (Operation *op : collector.loadOpInsts) {
node.loads.push_back(op);
- auto memref = cast<AffineReadOpInterface>(op).getMemRef();
+ auto memref =
+ canonicalizeMemref(cast<AffineReadOpInterface>(op).getMemRef());
memrefAccesses[memref].insert(node.id);
}
for (Operation *op : collector.storeOpInsts) {
node.stores.push_back(op);
- auto memref = cast<AffineWriteOpInterface>(op).getMemRef();
+ auto memref =
+ canonicalizeMemref(cast<AffineWriteOpInterface>(op).getMemRef());
memrefAccesses[memref].insert(node.id);
}
for (Operation *op : collector.memrefLoads) {
- SmallVector<Value> effectedValues;
- getEffectedValues<MemoryEffects::Read>(op, effectedValues);
- if (llvm::any_of(((ValueRange)effectedValues).getTypes(),
- [](Type type) { return !isa<MemRefType>(type); }))
- // We do not know the interaction here.
+ SmallVector<Value> affectedValues;
+ if (!getMayAffectedValues<MemoryEffects::Read>(op, affectedValues))
return nullptr;
- for (Value memref : effectedValues)
+ for (Value memref : affectedValues)
memrefAccesses[memref].insert(node.id);
node.memrefLoads.push_back(op);
}
for (Operation *op : collector.memrefStores) {
- SmallVector<Value> effectedValues;
- getEffectedValues<MemoryEffects::Write>(op, effectedValues);
- if (llvm::any_of((ValueRange(effectedValues)).getTypes(),
- [](Type type) { return !isa<MemRefType>(type); }))
+ SmallVector<Value> affectedValues;
+ if (!getMayAffectedValues<MemoryEffects::Write>(op, affectedValues))
return nullptr;
- for (Value memref : effectedValues)
+ for (Value memref : affectedValues)
memrefAccesses[memref].insert(node.id);
node.memrefStores.push_back(op);
}
for (Operation *op : collector.memrefFrees) {
- SmallVector<Value> effectedValues;
- getEffectedValues<MemoryEffects::Free>(op, effectedValues);
- if (llvm::any_of((ValueRange(effectedValues)).getTypes(),
- [](Type type) { return !isa<MemRefType>(type); }))
+ SmallVector<Value> affectedValues;
+ if (!getMayAffectedValues<MemoryEffects::Free>(op, affectedValues))
return nullptr;
- for (Value memref : effectedValues)
+ for (Value memref : affectedValues)
memrefAccesses[memref].insert(node.id);
node.memrefFrees.push_back(op);
}
@@ -243,17 +347,16 @@ addNodeToMDG(Operation *nodeOp, MemRefDependenceGraph &mdg,
return &node;
}
-/// Returns the memref being read/written by a memref/affine load/store op.
-static Value getMemRef(Operation *memOp) {
- if (auto memrefLoad = dyn_cast<memref::LoadOp>(memOp))
- return memrefLoad.getMemRef();
- if (auto affineLoad = dyn_cast<AffineReadOpInterface>(memOp))
- return affineLoad.getMemRef();
- if (auto memrefStore = dyn_cast<memref::StoreOp>(memOp))
- return memrefStore.getMemRef();
- if (auto affineStore = dyn_cast<AffineWriteOpInterface>(memOp))
- return affineStore.getMemRef();
- llvm_unreachable("unexpected op");
+/// Returns true if `op` may access `memref`, including through a view-like
+/// operation. Unknown operations are handled conservatively through their
+/// memory effects rather than assuming a particular operation class.
+static bool mayAccessMemRef(Operation *op, Value memref) {
+ if (auto affineRead = dyn_cast<AffineReadOpInterface>(op))
+ return isSameMemref(affineRead.getMemRef(), memref);
+ if (auto affineWrite = dyn_cast<AffineWriteOpInterface>(op))
+ return isSameMemref(affineWrite.getMemRef(), memref);
+ return mayHaveEffect<MemoryEffects::Read>(op, memref) ||
+ mayHaveEffect<MemoryEffects::Write>(op, memref);
}
/// Returns true if there may be a dependence on `memref` from srcNode's
@@ -267,21 +370,24 @@ static bool mayDependence(const Node &srcNode, const Node &dstNode,
assert(srcNode.op->getBlock() == dstNode.op->getBlock());
if (!isa<AffineForOp>(srcNode.op) || !isa<AffineForOp>(dstNode.op))
return true;
+ // Deallocation invalidates the whole storage object. Affine access
+ // relations cannot prove a free harmless by comparing indexed accesses.
+ if (srcNode.hasFree(memref) || dstNode.hasFree(memref))
+ return true;
// Conservatively handle dependences involving non-affine load/stores. Return
// true if there exists a conflicting read/write access involving such.
// Check whether there is a dependence from a source read/write op to a
- // destination read/write one; all expected to be memref/affine load/store.
+ // destination read/write one.
auto hasNonAffineDep = [&](ArrayRef<Operation *> srcMemOps,
ArrayRef<Operation *> dstMemOps) {
return llvm::any_of(srcMemOps, [&](Operation *srcOp) {
- Value srcMemref = getMemRef(srcOp);
- if (srcMemref != memref)
+ if (!mayAccessMemRef(srcOp, memref))
return false;
- return llvm::find_if(dstMemOps, [&](Operation *dstOp) {
- return srcMemref == getMemRef(dstOp);
- }) != dstMemOps.end();
+ return llvm::any_of(dstMemOps, [&](Operation *dstOp) {
+ return mayAccessMemRef(dstOp, memref);
+ });
});
};
@@ -318,13 +424,18 @@ static bool mayDependence(const Node &srcNode, const Node &dstNode,
for (auto *srcMemOp :
llvm::concat<Operation *const>(srcNode.stores, srcNode.loads)) {
MemRefAccess srcAcc(srcMemOp);
- if (srcAcc.memref != memref)
+ if (!isSameMemref(srcAcc.memref, memref))
continue;
for (auto *destMemOp :
llvm::concat<Operation *const>(dstNode.stores, dstNode.loads)) {
MemRefAccess destAcc(destMemOp);
- if (destAcc.memref != memref)
+ if (!isSameMemref(destAcc.memref, memref))
+ continue;
+ if (srcAcc.memref != destAcc.memref) {
+ if (srcAcc.isStore() || destAcc.isStore())
+ return true;
continue;
+ }
// Check for a top-level dependence between srcNode and destNode's ops.
if (!noDependence(checkMemrefAccessDependence(
srcAcc, destAcc, getNestingDepth(srcNode.op) + 1)))
@@ -337,7 +448,7 @@ static bool mayDependence(const Node &srcNode, const Node &dstNode,
bool MemRefDependenceGraph::init(bool fullAffineDependences) {
LDBG() << "--- Initializing MDG ---";
// Map from a memref to the set of ids of the nodes that have ops accessing
- // the memref.
+ // the memref. View-like values use their canonical storage source here.
DenseMap<Value, SetVector<unsigned>> memrefAccesses;
// Create graph nodes.
@@ -349,17 +460,23 @@ bool MemRefDependenceGraph::init(bool fullAffineDependences) {
return false;
forToNodeMap[&op] = node->id;
} else if (isa<AffineReadOpInterface>(op)) {
+ if (hasUnmodeledAffineMemoryEffects(&op))
+ return false;
// Create graph node for top-level load op.
Node node(nextNodeId++, &op);
node.loads.push_back(&op);
- auto memref = cast<AffineReadOpInterface>(op).getMemRef();
+ auto memref =
+ canonicalizeMemref(cast<AffineReadOpInterface>(op).getMemRef());
memrefAccesses[memref].insert(node.id);
nodes.insert({node.id, node});
} else if (isa<AffineWriteOpInterface>(op)) {
+ if (hasUnmodeledAffineMemoryEffects(&op))
+ return false;
// Create graph node for top-level store op.
Node node(nextNodeId++, &op);
node.stores.push_back(&op);
- auto memref = cast<AffineWriteOpInterface>(op).getMemRef();
+ auto memref =
+ canonicalizeMemref(cast<AffineWriteOpInterface>(op).getMemRef());
memrefAccesses[memref].insert(node.id);
nodes.insert({node.id, node});
} else if (op.getNumResults() > 0 && !op.use_empty()) {
@@ -417,7 +534,7 @@ bool MemRefDependenceGraph::init(bool fullAffineDependences) {
continue;
assert(forToNodeMap.count(*it) > 0 && "missing mapping");
unsigned userLoopNestId = forToNodeMap[*it];
- addEdge(node.id, userLoopNestId, value);
+ addEdge(node.id, userLoopNestId, value, Edge::Kind::SSA);
}
}
}
@@ -441,7 +558,7 @@ bool MemRefDependenceGraph::init(bool fullAffineDependences) {
// Check precise affine deps if asked for; otherwise, conservative.
if (!fullAffineDependences ||
mayDependence(*srcNode, *dstNode, srcMemRef))
- addEdge(srcId, dstId, srcMemRef);
+ addEdge(srcId, dstId, srcMemRef, Edge::Kind::Memory);
}
}
}
@@ -477,14 +594,14 @@ void MemRefDependenceGraph::removeNode(unsigned id) {
if (inEdges.count(id) > 0) {
SmallVector<Edge, 2> oldInEdges = inEdges[id];
for (auto &inEdge : oldInEdges) {
- removeEdge(inEdge.id, id, inEdge.value);
+ removeEdge(inEdge.id, id, inEdge.value, inEdge.kind);
}
}
// Remove each edge in 'outEdges[id]'.
if (outEdges.contains(id)) {
SmallVector<Edge, 2> oldOutEdges = outEdges[id];
for (auto &outEdge : oldOutEdges) {
- removeEdge(id, outEdge.id, outEdge.value);
+ removeEdge(id, outEdge.id, outEdge.value, outEdge.kind);
}
}
// Erase remaining node state.
@@ -529,40 +646,78 @@ bool MemRefDependenceGraph::hasEdge(unsigned srcId, unsigned dstId,
return hasOutEdge && hasInEdge;
}
+// Returns true iff there is an edge of 'kind' from node 'srcId' to node
+// 'dstId' which is for 'value' if non-null, or for any value otherwise.
+bool MemRefDependenceGraph::hasEdge(unsigned srcId, unsigned dstId, Value value,
+ Edge::Kind kind) const {
+ if (!outEdges.contains(srcId) || !inEdges.contains(dstId)) {
+ return false;
+ }
+ bool hasOutEdge = llvm::any_of(outEdges.lookup(srcId), [=](const Edge &edge) {
+ return edge.id == dstId && edge.kind == kind &&
+ (!value || edge.value == value);
+ });
+ bool hasInEdge = llvm::any_of(inEdges.lookup(dstId), [=](const Edge &edge) {
+ return edge.id == srcId && edge.kind == kind &&
+ (!value || edge.value == value);
+ });
+ return hasOutEdge && hasInEdge;
+}
+
// Adds an edge from node 'srcId' to node 'dstId' for 'value'.
-void MemRefDependenceGraph::addEdge(unsigned srcId, unsigned dstId,
- Value value) {
- if (!hasEdge(srcId, dstId, value)) {
- outEdges[srcId].push_back({dstId, value});
- inEdges[dstId].push_back({srcId, value});
- if (isa<MemRefType>(value.getType()))
- memrefEdgeCount[value]++;
+void MemRefDependenceGraph::addEdge(unsigned srcId, unsigned dstId, Value value,
+ Edge::Kind kind) {
+ assert(value && "dependence edges require a value");
+ if (!value)
+ return;
+ if (!hasEdge(srcId, dstId, value, kind)) {
+ outEdges[srcId].push_back({dstId, kind, value});
+ inEdges[dstId].push_back({srcId, kind, value});
+ // Allocation cleanup is storage-based, so both memory and SSA memref
+ // edges keep the canonical storage object live.
+ if (isa<BaseMemRefType>(value.getType()))
+ memrefEdgeCount[canonicalizeMemref(value)]++;
}
}
// Removes an edge from node 'srcId' to node 'dstId' for 'value'.
void MemRefDependenceGraph::removeEdge(unsigned srcId, unsigned dstId,
- Value value) {
- assert(inEdges.count(dstId) > 0);
- assert(outEdges.count(srcId) > 0);
- if (isa<MemRefType>(value.getType())) {
- assert(memrefEdgeCount.count(value) > 0);
- memrefEdgeCount[value]--;
- }
- // Remove 'srcId' from 'inEdges[dstId]'.
- for (auto *it = inEdges[dstId].begin(); it != inEdges[dstId].end(); ++it) {
- if ((*it).id == srcId && (*it).value == value) {
- inEdges[dstId].erase(it);
- break;
- }
- }
- // Remove 'dstId' from 'outEdges[srcId]'.
- for (auto *it = outEdges[srcId].begin(); it != outEdges[srcId].end(); ++it) {
- if ((*it).id == dstId && (*it).value == value) {
- outEdges[srcId].erase(it);
- break;
- }
+ Value value, Edge::Kind kind) {
+ assert(value && "dependence edges require a value");
+ if (!value)
+ return;
+ auto inEdgesIt = inEdges.find(dstId);
+ auto outEdgesIt = outEdges.find(srcId);
+ assert(inEdgesIt != inEdges.end() && outEdgesIt != outEdges.end() &&
+ "edge endpoints must exist");
+ if (inEdgesIt == inEdges.end() || outEdgesIt == outEdges.end())
+ return;
+
+ auto &inEdgesForDst = inEdgesIt->second;
+ auto &outEdgesForSrc = outEdgesIt->second;
+ auto inIt = llvm::find_if(inEdgesForDst, [&](const Edge &edge) {
+ return edge.id == srcId && edge.kind == kind && edge.value == value;
+ });
+ auto outIt = llvm::find_if(outEdgesForSrc, [&](const Edge &edge) {
+ return edge.id == dstId && edge.kind == kind && edge.value == value;
+ });
+ assert(inIt != inEdgesForDst.end() && outIt != outEdgesForSrc.end() &&
+ "edge must exist in both adjacency lists");
+ if (inIt == inEdgesForDst.end() || outIt == outEdgesForSrc.end())
+ return;
+
+ if (isa<BaseMemRefType>(value.getType())) {
+ Value canonicalValue = canonicalizeMemref(value);
+ auto countIt = memrefEdgeCount.find(canonicalValue);
+ assert(countIt != memrefEdgeCount.end() && countIt->second > 0 &&
+ "memref edge count must match the edge set");
+ if (countIt == memrefEdgeCount.end() || countIt->second == 0)
+ return;
+ --countIt->second;
}
+
+ inEdgesForDst.erase(inIt);
+ outEdgesForSrc.erase(outIt);
}
// Returns true if there is a path in the dependence graph from node 'srcId'
@@ -607,7 +762,7 @@ unsigned MemRefDependenceGraph::getIncomingMemRefAccesses(unsigned id,
Value memref) const {
unsigned inEdgeCount = 0;
for (const Edge &inEdge : inEdges.lookup(id)) {
- if (inEdge.value == memref) {
+ if (edgeMatchesMemref(inEdge, memref)) {
const Node *srcNode = getNode(inEdge.id);
// Only count in edges from 'srcNode' if 'srcNode' accesses 'memref'
if (srcNode->getStoreOpCount(memref) > 0)
@@ -623,7 +778,7 @@ unsigned MemRefDependenceGraph::getOutEdgeCount(unsigned id,
Value memref) const {
unsigned outEdgeCount = 0;
for (const auto &outEdge : outEdges.lookup(id))
- if (!memref || outEdge.value == memref)
+ if (!memref || edgeMatchesMemref(outEdge, memref))
++outEdgeCount;
return outEdgeCount;
}
@@ -632,10 +787,9 @@ unsigned MemRefDependenceGraph::getOutEdgeCount(unsigned id,
void MemRefDependenceGraph::gatherDefiningNodes(
unsigned id, DenseSet<unsigned> &definingNodes) const {
for (const Edge &edge : inEdges.lookup(id))
- // By definition of edge, if the edge value is a non-memref value,
- // then the dependence is between a graph node which defines an SSA value
- // and another graph node which uses the SSA value.
- if (!isa<MemRefType>(edge.value.getType()))
+ // SSA edges include memref definitions; memory edges are storage
+ // dependences even when their canonical value is a memref.
+ if (edge.kind == Edge::Kind::SSA)
definingNodes.insert(edge.id);
}
@@ -726,8 +880,10 @@ void MemRefDependenceGraph::updateEdges(unsigned srcId, unsigned dstId,
SmallVector<Edge, 2> oldInEdges = inEdges[srcId];
for (auto &inEdge : oldInEdges) {
// Add edge from 'inEdge.id' to 'dstId' if it's not a private memref.
- if (!privateMemRefs.contains(inEdge.value))
- addEdge(inEdge.id, dstId, inEdge.value);
+ if (!llvm::any_of(privateMemRefs, [&](Value privateMemRef) {
+ return edgeMatchesMemref(inEdge, privateMemRef);
+ }))
+ addEdge(inEdge.id, dstId, inEdge.value, inEdge.kind);
}
}
// For each edge in 'outEdges[srcId]': remove edge from 'srcId' to 'dstId'.
@@ -737,10 +893,10 @@ void MemRefDependenceGraph::updateEdges(unsigned srcId, unsigned dstId,
for (auto &outEdge : oldOutEdges) {
// Remove any out edges from 'srcId' to 'dstId' across memrefs.
if (outEdge.id == dstId)
- removeEdge(srcId, outEdge.id, outEdge.value);
+ removeEdge(srcId, outEdge.id, outEdge.value, outEdge.kind);
else if (removeSrcId) {
- addEdge(dstId, outEdge.id, outEdge.value);
- removeEdge(srcId, outEdge.id, outEdge.value);
+ addEdge(dstId, outEdge.id, outEdge.value, outEdge.kind);
+ removeEdge(srcId, outEdge.id, outEdge.value, outEdge.kind);
}
}
}
@@ -750,8 +906,10 @@ void MemRefDependenceGraph::updateEdges(unsigned srcId, unsigned dstId,
if (inEdges.count(dstId) > 0 && !privateMemRefs.empty()) {
SmallVector<Edge, 2> oldInEdges = inEdges[dstId];
for (auto &inEdge : oldInEdges)
- if (privateMemRefs.count(inEdge.value) > 0)
- removeEdge(inEdge.id, dstId, inEdge.value);
+ if (llvm::any_of(privateMemRefs, [&](Value privateMemRef) {
+ return edgeMatchesMemref(inEdge, privateMemRef);
+ }))
+ removeEdge(inEdge.id, dstId, inEdge.value, inEdge.kind);
}
}
@@ -764,8 +922,8 @@ void MemRefDependenceGraph::updateEdges(unsigned sibId, unsigned dstId) {
if (inEdges.count(sibId) > 0) {
SmallVector<Edge, 2> oldInEdges = inEdges[sibId];
for (auto &inEdge : oldInEdges) {
- addEdge(inEdge.id, dstId, inEdge.value);
- removeEdge(inEdge.id, sibId, inEdge.value);
+ addEdge(inEdge.id, dstId, inEdge.value, inEdge.kind);
+ removeEdge(inEdge.id, sibId, inEdge.value, inEdge.kind);
}
}
@@ -775,13 +933,13 @@ void MemRefDependenceGraph::updateEdges(unsigned sibId, unsigned dstId) {
if (outEdges.count(sibId) > 0) {
SmallVector<Edge, 2> oldOutEdges = outEdges[sibId];
for (auto &outEdge : oldOutEdges) {
- addEdge(dstId, outEdge.id, outEdge.value);
- removeEdge(sibId, outEdge.id, outEdge.value);
+ addEdge(dstId, outEdge.id, outEdge.value, outEdge.kind);
+ removeEdge(sibId, outEdge.id, outEdge.value, outEdge.kind);
}
}
}
-// Adds ops in 'loads' and 'stores' to node at 'id'.
+// Adds all collected memory operations to node at 'id'.
void MemRefDependenceGraph::addToNode(unsigned id, ArrayRef<Operation *> loads,
ArrayRef<Operation *> stores,
ArrayRef<Operation *> memrefLoads,
@@ -795,35 +953,39 @@ void MemRefDependenceGraph::addToNode(unsigned id, ArrayRef<Operation *> loads,
llvm::append_range(node->memrefFrees, memrefFrees);
}
-void MemRefDependenceGraph::clearNodeLoadAndStores(unsigned id) {
+void MemRefDependenceGraph::clearNodeMemoryOps(unsigned id) {
Node *node = getNode(id);
node->loads.clear();
node->stores.clear();
+ node->memrefLoads.clear();
+ node->memrefStores.clear();
+ node->memrefFrees.clear();
}
-// Calls 'callback' for each input edge incident to node 'id' which carries a
-// memref dependence.
+// Calls 'callback' for each input edge incident to node 'id' whose value is a
+// memref, whether it carries a memory or an SSA dependence.
void MemRefDependenceGraph::forEachMemRefInputEdge(
unsigned id, const std::function<void(Edge)> &callback) {
if (inEdges.count(id) > 0)
forEachMemRefEdge(inEdges.at(id), callback);
}
-// Calls 'callback' for each output edge from node 'id' which carries a
-// memref dependence.
+// Calls 'callback' for each output edge from node 'id' whose value is a
+// memref, whether it carries a memory or an SSA dependence.
void MemRefDependenceGraph::forEachMemRefOutputEdge(
unsigned id, const std::function<void(Edge)> &callback) {
if (outEdges.count(id) > 0)
forEachMemRefEdge(outEdges.at(id), callback);
}
-// Calls 'callback' for each edge in 'edges' which carries a memref
-// dependence.
+// Calls 'callback' for each edge in 'edges' whose value is a memref, whether it
+// carries a memory or an SSA dependence.
void MemRefDependenceGraph::forEachMemRefEdge(
ArrayRef<Edge> edges, const std::function<void(Edge)> &callback) {
for (const auto &edge : edges) {
- // Skip if 'edge' is not a memref dependence edge.
- if (!isa<MemRefType>(edge.value.getType()))
+ // Skip non-memref SSA edges; memory edges always carry a memref value.
+ if (edge.kind == Edge::Kind::SSA &&
+ !isa<BaseMemRefType>(edge.value.getType()))
continue;
assert(nodes.count(edge.id) > 0);
// Visit current input edge 'edge'.
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 1ec5fbfef50c3..431a335ffdeff 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -19,6 +19,7 @@
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/IR/AffineExpr.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/Builders.h"
@@ -154,7 +155,11 @@ static void getProducerCandidates(unsigned dstId,
if (any_of(srcNode->stores, [&](Operation *op) {
auto storeOp = cast<AffineWriteOpInterface>(op);
- return consumedMemrefs.count(storeOp.getMemRef()) > 0;
+ return llvm::any_of(consumedMemrefs, [&](Value consumedMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(consumedMemref),
+ cast<MemrefValue>(storeOp.getMemRef()));
+ });
}))
srcIdCandidates.push_back(srcNode->id);
}
@@ -218,7 +223,10 @@ static void gatherEscapingMemrefs(unsigned id, const MemRefDependenceGraph &mdg,
auto *node = mdg.getNode(id);
for (Operation *storeOp : node->stores) {
auto memref = cast<AffineWriteOpInterface>(storeOp).getMemRef();
- if (escapingMemRefs.count(memref))
+ if (llvm::any_of(escapingMemRefs, [&](Value escapingMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(escapingMemref), cast<MemrefValue>(memref));
+ }))
continue;
if (isEscapingMemref(memref, &mdg.block))
escapingMemRefs.insert(memref);
@@ -764,8 +772,9 @@ namespace {
// *) Update graph state to reflect the fusion of 'sibNode' into 'dstNode'.
//
// Given a graph where top-level operations are vertices in the set 'V' and
-// edges in the set 'E' are dependences between vertices, this algorithm
-// takes O(V) time for initialization, and has runtime O(V + E).
+// edges in the set 'E' are dependences between vertices, graph traversal is
+// linear in the graph size. Graph construction and fusion legality also
+// perform pairwise access and dependence checks for candidate loop depths.
//
// This greedy algorithm is not 'maximal' due to the current restriction of
// fusing along single producer consumer edges, but there is a TODO: to fix
@@ -852,7 +861,12 @@ struct GreedyFusion {
// 1. The source is to be removed after fusion,
// OR
// 2. The destination writes to `memref`.
- if (srcEscapingMemRefs.count(memref) > 0 &&
+ if (llvm::any_of(srcEscapingMemRefs,
+ [&](Value escapingMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(escapingMemref),
+ cast<MemrefValue>(memref));
+ }) &&
(removeSrcNode || consumerNode->getStoreOpCount(memref) > 0))
return false;
@@ -867,7 +881,12 @@ struct GreedyFusion {
// cannot create a private memref.
if (removeSrcNode &&
any_of(mdg->outEdges[producerId], [&](const auto &edge) {
- return edge.value == memref && edge.id != consumerId;
+ if (edge.id == consumerId)
+ return false;
+ if (edge.kind == MemRefDependenceGraph::Edge::Kind::SSA)
+ return edge.value == memref;
+ return memref::isSameViewOrTrivialAlias(cast<MemrefValue>(edge.value),
+ cast<MemrefValue>(memref));
}))
return false;
@@ -972,12 +991,20 @@ struct GreedyFusion {
// producer-consumer loads/stores.
SmallVector<Operation *, 2> dstMemrefOps;
for (Operation *op : dstNode->loads)
- if (producerConsumerMemrefs.count(
- cast<AffineReadOpInterface>(op).getMemRef()) > 0)
+ if (llvm::any_of(producerConsumerMemrefs, [&](Value producerMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(producerMemref),
+ cast<MemrefValue>(
+ cast<AffineReadOpInterface>(op).getMemRef()));
+ }))
dstMemrefOps.push_back(op);
for (Operation *op : dstNode->stores)
- if (producerConsumerMemrefs.count(
- cast<AffineWriteOpInterface>(op).getMemRef()))
+ if (llvm::any_of(producerConsumerMemrefs, [&](Value producerMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(producerMemref),
+ cast<MemrefValue>(
+ cast<AffineWriteOpInterface>(op).getMemRef()));
+ }))
dstMemrefOps.push_back(op);
if (dstMemrefOps.empty())
continue;
@@ -1050,8 +1077,13 @@ struct GreedyFusion {
// Retrieve producer stores from the src loop.
SmallVector<Operation *, 2> producerStores;
for (Operation *op : srcNode->stores)
- if (producerConsumerMemrefs.count(
- cast<AffineWriteOpInterface>(op).getMemRef()))
+ if (llvm::any_of(
+ producerConsumerMemrefs, [&](Value producerMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(producerMemref),
+ cast<MemrefValue>(
+ cast<AffineWriteOpInterface>(op).getMemRef()));
+ }))
producerStores.push_back(op);
assert(!producerStores.empty() && "Expected producer store");
@@ -1075,14 +1107,13 @@ struct GreedyFusion {
srcId, dstId, bestSlice, fusedLoopInsPoint, srcEscapingMemRefs,
*mdg);
- DenseSet<Value> privateMemrefs;
+ DenseSet<Value> privateMemrefsToCreate;
for (Value memref : producerConsumerMemrefs) {
if (canCreatePrivateMemRef(memref, srcEscapingMemRefs, srcId, dstId,
removeSrcNode)) {
// Create a private version of this memref.
LDBG() << "Creating private memref for " << memref;
- // Create a private version of this memref.
- privateMemrefs.insert(memref);
+ privateMemrefsToCreate.insert(memref);
}
}
@@ -1098,12 +1129,11 @@ struct GreedyFusion {
if (fusedLoopInsPoint != dstAffineForOp)
dstAffineForOp->moveBefore(fusedLoopInsPoint);
- // Update edges between 'srcNode' and 'dstNode'.
- mdg->updateEdges(srcNode->id, dstNode->id, privateMemrefs,
- removeSrcNode);
-
- // Create private memrefs.
- if (!privateMemrefs.empty()) {
+ // Create private memrefs. Only record a memref for edge updates after
+ // its replacement has succeeded; a failed private-buffer creation
+ // leaves the original memref uses and their dependences intact.
+ DenseSet<Value> privateMemrefs;
+ if (!privateMemrefsToCreate.empty()) {
// Note the block into which fusion was performed. This can be used to
// place `alloc`s that create private memrefs.
Block *sliceInsertionBlock = bestSlice.insertPoint->getBlock();
@@ -1112,7 +1142,11 @@ struct GreedyFusion {
DenseMap<Value, SmallVector<Operation *, 4>> privateMemRefToStores;
dstAffineForOp.walk([&](AffineWriteOpInterface storeOp) {
Value storeMemRef = storeOp.getMemRef();
- if (privateMemrefs.count(storeMemRef) > 0)
+ if (llvm::any_of(privateMemrefsToCreate, [&](Value privateMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(privateMemref),
+ cast<MemrefValue>(storeMemRef));
+ }))
privateMemRefToStores[storeMemRef].push_back(storeOp);
});
@@ -1121,16 +1155,21 @@ struct GreedyFusion {
// loads and stores. Any reference to the original ones becomes
// invalid after this point.
for (auto &memrefToStoresPair : privateMemRefToStores) {
+ // Capture the pre-replacement value; the store operands are
+ // rewritten before createPrivateMemRef returns.
+ Value oldMemRef = memrefToStoresPair.first;
ArrayRef<Operation *> storesForMemref = memrefToStoresPair.second;
Value newMemRef = createPrivateMemRef(
dstAffineForOp, storesForMemref, bestDstLoopDepth,
fastMemorySpace, sliceInsertionBlock, localBufSizeThreshold);
if (!newMemRef)
continue;
+ privateMemrefs.insert(oldMemRef);
// Create new node in dependence graph for 'newMemRef' alloc op.
unsigned newMemRefNodeId = mdg->addNode(newMemRef.getDefiningOp());
// Add edge from 'newMemRef' node to dstNode.
- mdg->addEdge(newMemRefNodeId, dstId, newMemRef);
+ mdg->addEdge(newMemRefNodeId, dstId, newMemRef,
+ MemRefDependenceGraph::Edge::Kind::SSA);
}
// One or more entries for 'newMemRef' alloc op are inserted into
// the DenseMap mdg->nodes. Since an insertion may cause DenseMap to
@@ -1138,12 +1177,16 @@ struct GreedyFusion {
dstNode = mdg->getNode(dstId);
}
+ // Update edges between 'srcNode' and 'dstNode' after IR mutation and
+ // only for private memrefs whose replacements succeeded.
+ mdg->updateEdges(srcId, dstId, privateMemrefs, removeSrcNode);
+
// Collect dst loop stats after memref privatization transformation.
LoopNestStateCollector dstLoopCollector;
dstLoopCollector.collect(dstAffineForOp);
- // Clear and add back loads and stores.
- mdg->clearNodeLoadAndStores(dstNode->id);
+ // Clear and add back all memory operations.
+ mdg->clearNodeMemoryOps(dstNode->id);
mdg->addToNode(
dstId, dstLoopCollector.loadOpInsts, dstLoopCollector.storeOpInsts,
dstLoopCollector.memrefLoads, dstLoopCollector.memrefStores,
@@ -1387,7 +1430,8 @@ struct GreedyFusion {
DenseSet<Value> storeMemrefs;
for (auto *storeOpInst : sibNode->stores) {
storeMemrefs.insert(
- cast<AffineWriteOpInterface>(storeOpInst).getMemRef());
+ memref::skipFullyAliasingOperations(cast<MemrefValue>(
+ cast<AffineWriteOpInterface>(storeOpInst).getMemRef())));
}
return storeMemrefs.size() <= 1;
};
@@ -1457,7 +1501,10 @@ struct GreedyFusion {
if (visitedSibNodeIds->count(sibNodeId) > 0)
return;
// Skip output edge if not a sibling using the same memref.
- if (outEdge.id == dstNode->id || outEdge.value != inEdge.value)
+ if (outEdge.id == dstNode->id ||
+ !memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(outEdge.value),
+ cast<MemrefValue>(inEdge.value)))
return;
auto *sibNode = mdg->getNode(sibNodeId);
if (!isa<AffineForOp>(sibNode->op))
@@ -1490,8 +1537,8 @@ struct GreedyFusion {
auto dstForInst = cast<AffineForOp>(dstNode->op);
LoopNestStateCollector dstLoopCollector;
dstLoopCollector.collect(dstForInst);
- // Clear and add back loads and stores
- mdg->clearNodeLoadAndStores(dstNode->id);
+ // Clear and add back all memory operations.
+ mdg->clearNodeMemoryOps(dstNode->id);
mdg->addToNode(dstNode->id, dstLoopCollector.loadOpInsts,
dstLoopCollector.storeOpInsts, dstLoopCollector.memrefLoads,
dstLoopCollector.memrefStores, dstLoopCollector.memrefFrees);
diff --git a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt
index ef6e0dbf45d3a..efeac83a098e7 100644
--- a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_dialect_library(MLIRAffineUtils
MLIRArithUtils
MLIRFuncDialect
MLIRMemRefDialect
+ MLIRMemRefUtils
MLIRTransformUtils
MLIRViewLikeInterface
)
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
index 68296ea3368a1..6c45a0ad2382d 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
@@ -18,6 +18,7 @@
#include "mlir/Dialect/Affine/Analysis/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/Operation.h"
#include "mlir/IR/PatternMatch.h"
@@ -37,10 +38,14 @@ static void getLoadAndStoreMemRefAccesses(Operation *opA,
DenseMap<Value, bool> &values) {
opA->walk([&](Operation *op) {
if (auto loadOp = dyn_cast<AffineReadOpInterface>(op)) {
- if (values.count(loadOp.getMemRef()) == 0)
- values[loadOp.getMemRef()] = false;
+ Value memref = memref::skipFullyAliasingOperations(
+ cast<MemrefValue>(loadOp.getMemRef()));
+ if (values.count(memref) == 0)
+ values[memref] = false;
} else if (auto storeOp = dyn_cast<AffineWriteOpInterface>(op)) {
- values[storeOp.getMemRef()] = true;
+ Value memref = memref::skipFullyAliasingOperations(
+ cast<MemrefValue>(storeOp.getMemRef()));
+ values[memref] = true;
}
});
}
@@ -50,10 +55,16 @@ static void getLoadAndStoreMemRefAccesses(Operation *opA,
/// Returns false otherwise.
static bool isDependentLoadOrStoreOp(Operation *op,
DenseMap<Value, bool> &values) {
- if (auto loadOp = dyn_cast<AffineReadOpInterface>(op))
- return values.count(loadOp.getMemRef()) > 0 && values[loadOp.getMemRef()];
- if (auto storeOp = dyn_cast<AffineWriteOpInterface>(op))
- return values.count(storeOp.getMemRef()) > 0;
+ if (auto loadOp = dyn_cast<AffineReadOpInterface>(op)) {
+ Value memref = memref::skipFullyAliasingOperations(
+ cast<MemrefValue>(loadOp.getMemRef()));
+ return values.count(memref) > 0 && values[memref];
+ }
+ if (auto storeOp = dyn_cast<AffineWriteOpInterface>(op)) {
+ Value memref = memref::skipFullyAliasingOperations(
+ cast<MemrefValue>(storeOp.getMemRef()));
+ return values.count(memref) > 0;
+ }
return false;
}
@@ -180,6 +191,63 @@ gatherLoadsAndStores(AffineForOp forOp,
return !hasIfOp;
}
+// The fusion transformation clones the complete source loop body into the
+// destination schedule. The affine slice and dependence analysis below does
+// not model the order or execution count of non-affine effects within that
+// schedule, so reject every such effect here. This check is intentionally
+// independent of MDG construction because this utility is also used as a
+// partial predicate by tests and by the main pass after MDG legality checks.
+// It does not model generic or unknown effects in operations between the
+// candidate loops; the MDG-backed caller remains responsible for those.
+static bool hasUnmodeledMemoryEffects(AffineForOp forOp) {
+ LoopNestStateCollector collector;
+ collector.collect(forOp);
+ if (collector.hasUnmodeledMemoryEffects)
+ return true;
+
+ bool hasEffects = false;
+ forOp.walk([&](Operation *op) {
+ if (hasEffects || isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
+ return;
+ auto memInterface = dyn_cast<MemoryEffectOpInterface>(op);
+ if (!memInterface) {
+ hasEffects = hasUnknownEffects(op);
+ return;
+ }
+
+ SmallVector<SideEffects::EffectInstance<MemoryEffects::Effect>, 4> effects;
+ memInterface.getEffects(effects);
+ hasEffects = !effects.empty();
+ });
+ return hasEffects;
+}
+
+// Returns true when accesses in two loop bodies use different views of the
+// same storage and at least one access writes. The raw view remains important
+// for affine coordinate analysis, but a storage-level conflict is enough to
+// reject fusion because this check protects the frame condition of the
+// complete loop bodies before strategy-specific filtering.
+static bool hasConflictingStorageAccesses(ArrayRef<Operation *> firstOps,
+ ArrayRef<Operation *> secondOps) {
+ for (Operation *firstOp : firstOps) {
+ MemRefAccess firstAccess(firstOp);
+ Value firstStorage =
+ memref::skipViewLikeOps(cast<MemrefValue>(firstAccess.memref));
+ for (Operation *secondOp : secondOps) {
+ MemRefAccess secondAccess(secondOp);
+ if (firstAccess.memref == secondAccess.memref)
+ continue;
+ Value secondStorage =
+ memref::skipViewLikeOps(cast<MemrefValue>(secondAccess.memref));
+ if (firstStorage != secondStorage)
+ continue;
+ if (firstAccess.isStore() || secondAccess.isStore())
+ return true;
+ }
+ }
+ return false;
+}
+
/// Returns the maximum loop depth at which we could fuse producer loop
/// 'srcForOp' into consumer loop 'dstForOp' without violating data dependences.
// TODO: Generalize this check for sibling and more generic fusion scenarios.
@@ -200,7 +268,10 @@ static unsigned getMaxLoopDepth(ArrayRef<Operation *> srcOps,
auto loadOp = dyn_cast<AffineReadOpInterface>(dstOp);
Value memref = loadOp ? loadOp.getMemRef()
: cast<AffineWriteOpInterface>(dstOp).getMemRef();
- if (producerConsumerMemrefs.count(memref) > 0)
+ if (llvm::any_of(producerConsumerMemrefs, [&](Value producerMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(producerMemref), cast<MemrefValue>(memref));
+ }))
targetDstOps.push_back(dstOp);
}
@@ -223,13 +294,26 @@ static unsigned getMaxLoopDepth(ArrayRef<Operation *> srcOps,
auto *dstOpInst = targetDstOps[j];
MemRefAccess dstAccess(dstOpInst);
+ if (!memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(srcAccess.memref),
+ cast<MemrefValue>(dstAccess.memref)))
+ continue;
+ // Affine maps are expressed in the raw view coordinates. Do not run
+ // precise dependence analysis across different views without a
+ // translation between those coordinate systems.
+ if (srcAccess.memref != dstAccess.memref) {
+ if (srcAccess.isStore() || dstAccess.isStore())
+ return 0;
+ continue;
+ }
+
unsigned numCommonLoops =
getNumCommonSurroundingLoops(*srcOpInst, *dstOpInst);
for (unsigned d = 1; d <= numCommonLoops + 1; ++d) {
// TODO: Cache dependence analysis results, check cache here.
DependenceResult result =
checkMemrefAccessDependence(srcAccess, dstAccess, d);
- if (hasDependence(result)) {
+ if (mayHaveDependence(result)) {
// Store minimum loop depth and break because we want the min 'd' at
// which there is a dependence.
loopDepth = std::min(loopDepth, d - 1);
@@ -262,6 +346,12 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
return FusionResult::FailPrecondition;
}
+ if (hasUnmodeledMemoryEffects(srcForOp) ||
+ hasUnmodeledMemoryEffects(dstForOp)) {
+ LDBG() << "Cannot fuse loop nests with unmodeled non-affine memory effects";
+ return FusionResult::FailFusionDependence;
+ }
+
// Return 'failure' if no valid insertion point for fused loop nest in 'block'
// exists which would preserve dependences.
if (!getFusedLoopNestInsertionPoint(srcForOp, dstForOp)) {
@@ -289,6 +379,12 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
return FusionResult::FailPrecondition;
}
+ if (hasConflictingStorageAccesses(opsA, opsB)) {
+ LDBG() << "Fusion would change ordering of accesses through different "
+ "views of the same storage";
+ return FusionResult::FailFusionDependence;
+ }
+
// Return 'failure' if fusing loops at depth 'dstLoopDepth' wouldn't preserve
// loop dependences.
// TODO: Enable this check for sibling and more generic loop fusion
@@ -328,12 +424,36 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
// to 'memref' in 'srcForOp' to compute the slice union.
for (Operation *op : opsA) {
auto load = dyn_cast<AffineReadOpInterface>(op);
- if (load && load.getMemRef() == fusionStrategy.getSiblingFusionMemRef())
+ if (load &&
+ memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(load.getMemRef()),
+ cast<MemrefValue>(fusionStrategy.getSiblingFusionMemRef())))
strategyOpsA.push_back(op);
}
break;
}
+ // Affine access maps are expressed in the raw view coordinates. Reject a
+ // fusion that would pair different views from one trivial alias class until
+ // a coordinate translation is available.
+ auto getMemref = [](Operation *op) -> Value {
+ if (auto load = dyn_cast<AffineReadOpInterface>(op))
+ return load.getMemRef();
+ return cast<AffineWriteOpInterface>(op).getMemRef();
+ };
+ if (llvm::any_of(strategyOpsA, [&](Operation *srcOp) {
+ return llvm::any_of(opsB, [&](Operation *dstOp) {
+ Value srcMemref = getMemref(srcOp);
+ Value dstMemref = getMemref(dstOp);
+ return srcMemref != dstMemref &&
+ memref::isSameViewOrTrivialAlias(cast<MemrefValue>(srcMemref),
+ cast<MemrefValue>(dstMemref));
+ });
+ })) {
+ LDBG() << "Fusion across different trivial alias views is unsupported";
+ return FusionResult::FailFusionDependence;
+ }
+
// Compute union of computation slices computed between all pairs of ops
// from 'forOpA' and 'forOpB'.
SliceComputationResult sliceComputationResult = affine::computeSliceUnion(
@@ -652,6 +772,10 @@ void mlir::affine::gatherProducerConsumerMemrefs(
// memrefs from loads in 'dstOps'.
for (Operation *op : dstOps)
if (auto loadOp = dyn_cast<AffineReadOpInterface>(op))
- if (srcStoreMemRefs.count(loadOp.getMemRef()) > 0)
+ if (llvm::any_of(srcStoreMemRefs, [&](Value storeMemref) {
+ return memref::isSameViewOrTrivialAlias(
+ cast<MemrefValue>(storeMemref),
+ cast<MemrefValue>(loadOp.getMemRef()));
+ }))
producerConsumerMemrefs.insert(loadOp.getMemRef());
}
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 90bc57e950cf1..2f38decb3c81c 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -18,6 +18,7 @@
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/IntegerSet.h"
@@ -1357,7 +1358,8 @@ bool mlir::affine::isValidLoopInterchangePermutation(
// Gather dependence components for dependences between all ops in loop nest
// rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
std::vector<SmallVector<DependenceComponent, 2>> depCompsVec;
- getDependenceComponents(loops[0], maxLoopDepth, &depCompsVec);
+ if (failed(getDependenceComponents(loops[0], maxLoopDepth, &depCompsVec)))
+ return false;
return checkLoopInterchangeDependences(depCompsVec, loops, loopPermMap);
}
@@ -1453,6 +1455,31 @@ unsigned mlir::affine::permuteLoops(ArrayRef<AffineForOp> input,
return invPermMap[0].second;
}
+// Returns true when affine accesses use different views of the same storage
+// and at least one access writes. The affine dependence analysis cannot
+// compare access maps expressed in different view coordinate systems, so a
+// schedule-changing transform must conservatively preserve this ordering.
+static bool
+hasConflictingAffineStorageAccesses(ArrayRef<Operation *> affineAccesses) {
+ for (unsigned i = 0, e = affineAccesses.size(); i < e; ++i) {
+ MemRefAccess firstAccess(affineAccesses[i]);
+ Value firstStorage =
+ memref::skipViewLikeOps(cast<MemrefValue>(firstAccess.memref));
+ for (unsigned j = i + 1; j < e; ++j) {
+ MemRefAccess secondAccess(affineAccesses[j]);
+ if (firstAccess.memref == secondAccess.memref)
+ continue;
+ Value secondStorage =
+ memref::skipViewLikeOps(cast<MemrefValue>(secondAccess.memref));
+ if (firstStorage != secondStorage)
+ continue;
+ if (firstAccess.isStore() || secondAccess.isStore())
+ return true;
+ }
+ }
+ return false;
+}
+
// Sinks all sequential loops to the innermost levels (while preserving
// relative order among them) and moves all parallel loops to the
// outermost (while again preserving relative order among them).
@@ -1462,11 +1489,39 @@ AffineForOp mlir::affine::sinkSequentialLoops(AffineForOp forOp) {
if (loops.size() < 2)
return forOp;
+ // Dependence components only model affine memory accesses. Do not reorder a
+ // loop nest containing any other memory effect, even if that effect can be
+ // represented by a different analysis, because its schedule is not covered
+ // by the dependence components used below.
+ LoopNestStateCollector collector;
+ collector.collect(loops[0]);
+ if (collector.hasUnmodeledMemoryEffects)
+ return forOp;
+ SmallVector<Operation *, 8> affineAccesses;
+ for (Operation *op : collector.loadOpInsts)
+ affineAccesses.push_back(op);
+ for (Operation *op : collector.storeOpInsts)
+ affineAccesses.push_back(op);
+ if (hasConflictingAffineStorageAccesses(affineAccesses))
+ return forOp;
+ if (loops[0]
+ ->walk([&](Operation *op) {
+ if (isa<AffineForOp, AffineReadOpInterface, AffineWriteOpInterface>(
+ op) ||
+ op->mightHaveTrait<OpTrait::IsTerminator>())
+ return WalkResult::advance();
+ return isMemoryEffectFree(op) ? WalkResult::advance()
+ : WalkResult::interrupt();
+ })
+ .wasInterrupted())
+ return forOp;
+
// Gather dependence components for dependences between all ops in loop nest
// rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
unsigned maxLoopDepth = loops.size();
std::vector<SmallVector<DependenceComponent, 2>> depCompsVec;
- getDependenceComponents(loops[0], maxLoopDepth, &depCompsVec);
+ if (failed(getDependenceComponents(loops[0], maxLoopDepth, &depCompsVec)))
+ return forOp;
// Mark loops as either parallel or sequential.
SmallVector<bool, 8> isParallelLoop(maxLoopDepth, true);
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 7043083298615..ec69aba8619f5 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -646,7 +646,7 @@ static bool mustReachAtInnermost(const MemRefAccess &srcAccess,
getNumCommonSurroundingLoops(*srcAccess.opInst, *destAccess.opInst);
DependenceResult result =
checkMemrefAccessDependence(srcAccess, destAccess, nsLoops + 1);
- return hasDependence(result);
+ return mustHaveDependence(result);
}
/// Returns true if `srcMemOp` may have an effect on `destMemOp` within the
diff --git a/mlir/test/Dialect/Affine/loop-fusion-2.mlir b/mlir/test/Dialect/Affine/loop-fusion-2.mlir
index b26a539b2f7d5..2ef55b01164e2 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-2.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-2.mlir
@@ -20,15 +20,15 @@ func.func @should_fuse_at_depth_above_loop_carried_dependence(%arg0: memref<64x4
affine.for %i3 = 0 to 4 {
affine.for %i4 = 0 to 16 {
%v = affine.load %arg1[16 * %i3 - %i4 + 15, %i2] : memref<64x4xf32>
- "op0"(%v) : (f32) -> ()
+ %unused0 = arith.addf %v, %v : f32
}
affine.for %i5 = 0 to 4 {
affine.for %i6 = 0 to 16 {
%v = affine.load %arg0[16 * %i5 - %i6 + 15, %i3] : memref<64x4xf32>
- "op1"(%v) : (f32) -> ()
+ %unused1 = arith.addf %v, %v : f32
}
affine.for %i7 = 0 to 16 {
- %r = "op2"() : () -> (f32)
+ %r = arith.constant 0.0 : f32
%v = affine.load %out[16 * %i5 + %i7, %i2] : memref<64x4xf32>
%s = arith.addf %v, %r : f32
affine.store %s, %out[16 * %i5 + %i7, %i2] : memref<64x4xf32>
@@ -57,15 +57,15 @@ func.func @should_fuse_at_depth_above_loop_carried_dependence(%arg0: memref<64x4
// CHECK-NEXT: affine.for %{{.*}} = 0 to 4 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}} * 16 - %{{.*}} + 15, %{{.*}}] : memref<64x4xf32>
- // CHECK-NEXT: "op0"(%{{.*}}) : (f32) -> ()
+ // CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 4 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}} * 16 - %{{.*}} + 15, %{{.*}}] : memref<64x4xf32>
- // CHECK-NEXT: "op1"(%{{.*}}) : (f32) -> ()
+ // CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
- // CHECK-NEXT: %{{.*}} = "op2"() : () -> f32
+ // CHECK-NEXT: %{{.*}} = arith.constant {{.*}} : f32
// CHECK: affine.load %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<64x4xf32>
// CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK: affine.store %{{.*}}, %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<64x4xf32>
@@ -244,7 +244,7 @@ func.func @slice_tile(%arg0: memref<128x8xf32>, %arg1: memref<32x8xf32>, %0 : f3
affine.for %k = 0 to 8 {
affine.for %kk = 0 to 16 {
%v = affine.load %arg0[16 * %k + %kk, %j] : memref<128x8xf32>
- %r = "foo"(%v) : (f32) -> f32
+ %r = arith.addf %v, %v : f32
}
affine.for %ii = 0 to 16 {
%v = affine.load %arg1[16 * %i + %ii, %j] : memref<32x8xf32>
@@ -264,7 +264,7 @@ func.func @slice_tile(%arg0: memref<128x8xf32>, %arg1: memref<32x8xf32>, %0 : f3
// CHECK-NEXT: affine.for %{{.*}} = 0 to 8 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<128x8xf32>
-// CHECK-NEXT: "foo"(%{{.*}}) : (f32) -> f32
+// CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<32x8xf32>
@@ -463,26 +463,26 @@ func.func @should_not_slice_past_slice_barrier() {
%0 = memref.alloc() : memref<100x16xf32>
affine.for %i0 = 0 to 100 {
affine.for %i1 = 0 to 16 {
- %1 = "op1"() : () -> f32
+ %1 = arith.constant 0.0 : f32
affine.store %1, %0[%i0, %i1] : memref<100x16xf32>
} {slice_fusion_barrier = true}
}
affine.for %i2 = 0 to 100 {
affine.for %i3 = 0 to 16 {
%2 = affine.load %0[%i2, %i3] : memref<100x16xf32>
- "op2"(%2) : (f32) -> ()
+ %unused = arith.addf %2, %2 : f32
}
}
// The 'slice_fusion_barrier' attribute on '%i1' prevents slicing the
// iteration space of '%i1' and any enclosing loop nests.
// CHECK: affine.for %{{.*}} = 0 to 100 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
-// CHECK-NEXT: %{{.*}} = "op1"() : () -> f32
+// CHECK-NEXT: %{{.*}} = arith.constant {{.*}} : f32
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0, %{{.*}}] : memref<1x16xf32>
// CHECK-NEXT: } {slice_fusion_barrier = true}
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[0, %{{.*}}] : memref<1x16xf32>
-// CHECK-NEXT: "op2"(%{{.*}}) : (f32) -> ()
+// CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: }
return
diff --git a/mlir/test/Dialect/Affine/loop-fusion-3.mlir b/mlir/test/Dialect/Affine/loop-fusion-3.mlir
index 70d6c82105543..d8204d0afc376 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-3.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-3.mlir
@@ -869,7 +869,8 @@ func.func @call_op_prevents_fusion(%arg0: memref<16xf32>){
// -----
func.func private @some_function()
-func.func @call_op_does_not_prevent_fusion(%arg0: memref<16xf32>){
+func.func @call_op_without_memref_operands_prevents_fusion(
+ %arg0: memref<16xf32>) {
%A = memref.alloc() : memref<16xf32>
%cst_1 = arith.constant 1.000000e+00 : f32
affine.for %arg1 = 0 to 16 {
@@ -885,9 +886,10 @@ func.func @call_op_does_not_prevent_fusion(%arg0: memref<16xf32>){
}
return
}
-// CHECK-LABEL: func @call_op_does_not_prevent_fusion
+// CHECK-LABEL: func @call_op_without_memref_operands_prevents_fusion
+// CHECK: affine.for
+// CHECK: call @some_function() : () -> ()
// CHECK: affine.for
-// CHECK-NOT: affine.for
// -----
@@ -1281,14 +1283,13 @@ func.func @unknown_memref_def_op() {
affine.for %i1 = 0 to 10 {
%0 = affine.load %may_alias[%i1] : memref<10xf32>
}
- // Fusion happens, but memref isn't privatized since %may_alias's origin is
- // unknown.
+ // The unknown call prevents fusion because its memory effects are not
+ // limited to the returned memref.
// CHECK: call
// CHECK-NEXT: affine.for
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+ // CHECK: affine.for
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
- // CHECK-NEXT: }
- // CHECK-NOT: affine.for
return
}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-4.mlir b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
index cf530016c201a..b747e8e8b7b8b 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-4.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
@@ -1,6 +1,7 @@
// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion{mode=producer}))' -split-input-file | FileCheck %s --check-prefix=PRODUCER-CONSUMER
// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion{compute-tolerance=0.0}))' -split-input-file | FileCheck %s --check-prefix=ZERO-TOLERANCE
// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion{mode=producer maximal}))' -split-input-file | FileCheck %s --check-prefix=PRODUCER-CONSUMER-MAXIMAL
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion{mode=producer maximal}))' -split-input-file | FileCheck %s --check-prefix=NO-REORDER
// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion{maximal mode=sibling}))' -split-input-file | FileCheck %s --check-prefix=SIBLING-MAXIMAL
// All fusion: producer-consumer and sibling.
// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(affine-loop-fusion))' -split-input-file | FileCheck %s --check-prefix=ALL
@@ -364,7 +365,8 @@ func.func @same_memref_load_multiple_stores(%producer : memref<32xf32>, %produce
#map = affine_map<()[s0] -> (s0 + 5)>
#map1 = affine_map<()[s0] -> (s0 + 17)>
-// Test with non-int/float memref types.
+// Test with non-int/float memref types. The non-affine memref.load in the
+// consumer is not modeled by the affine slice analysis, so fusion is refused.
// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @memref_index_type
func.func @memref_index_type() {
@@ -388,15 +390,41 @@ func.func @memref_index_type() {
%7 = memref.load %alloc[%5, %6] : memref<8x18xf32>
affine.store %7, %alloc_1[%arg3] : memref<3xf32>
}
- // Expect fusion.
+ // Do not fuse a candidate loop containing a non-affine memory effect.
// PRODUCER-CONSUMER-MAXIMAL: affine.for
- // PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+ // PRODUCER-CONSUMER-MAXIMAL: affine.store
+ // PRODUCER-CONSUMER-MAXIMAL: }
+ // PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+ // PRODUCER-CONSUMER-MAXIMAL: memref.load
// PRODUCER-CONSUMER-MAXIMAL: return
return
}
// -----
+// Keep a positive affine-only case for non-float element types. The element
+// type itself is not a reason to reject fusion.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @memref_index_element_type_fuses
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: return
+func.func @memref_index_element_type_fuses(
+ %in: memref<3xindex>, %out: memref<3xindex>) {
+ %tmp = memref.alloc() : memref<3xindex>
+ affine.for %i = 0 to 3 {
+ %v = affine.load %in[%i] : memref<3xindex>
+ affine.store %v, %tmp[%i] : memref<3xindex>
+ }
+ affine.for %j = 0 to 3 {
+ %v = affine.load %tmp[%j] : memref<3xindex>
+ affine.store %v, %out[%j] : memref<3xindex>
+ }
+ return
+}
+
+// -----
+
#map = affine_map<(d0) -> (d0)>
#map1 =affine_map<(d0) -> (d0 + 1)>
@@ -583,11 +611,15 @@ func.func @zero_tolerance(%arg0: memref<65536xcomplex<f64>>, %arg1: memref<30x13
affine.store %18, %2[%arg2] : memref<131072xi128>
affine.store %13, %1[%arg2] : memref<131072xi1>
}
- // The next two nests are fused.
+ // The next nest cannot fuse with the one following it across the opaque
+ // external call below, which may write to its memref operand in place.
// ZERO-TOLERANCE: affine.for %{{.*}} = 0 to 30
// ZERO-TOLERANCE-NEXT: affine.for %{{.*}} = 0 to 131072
// ZERO-TOLERANCE: func.call @__external_reduce_barrett
// ZERO-TOLERANCE: affine.store
+ // ZERO-TOLERANCE: call @__external_levelwise_forward_ntt
+ // ZERO-TOLERANCE-NEXT: affine.for %{{.*}} = 0 to 30
+ // ZERO-TOLERANCE-NEXT: affine.for %{{.*}} = 0 to 131072
// ZERO-TOLERANCE: affine.load
// ZERO-TOLERANCE-NEXT: affine.store
affine.for %arg2 = 0 to 30 {
@@ -611,9 +643,14 @@ func.func @zero_tolerance(%arg0: memref<65536xcomplex<f64>>, %arg1: memref<30x13
affine.store %7, %arg1[%arg2, %arg3] : memref<30x131072xi64>
}
}
- // Under maximal fusion, just one nest.
+ // Under maximal fusion, the first two nests fuse, but the last nest cannot
+ // fuse into them across the opaque external call, which may write to its
+ // memref operand in place.
// PRODUCER-CONSUMER-MAXIMAL: affine.for %{{.*}} = 0 to 30
// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for %{{.*}} = 0 to 131072
+ // PRODUCER-CONSUMER-MAXIMAL: call @__external_levelwise_forward_ntt
+ // PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for %{{.*}} = 0 to 30
+ // PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for %{{.*}} = 0 to 131072
// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for %{{.*}}
memref.dealloc %2 : memref<131072xi128>
memref.dealloc %1 : memref<131072xi1>
@@ -884,3 +921,822 @@ func.func @high_trip_count(%arg0: memref<1024x4096xf32>, %arg1: memref<8192x4096
}
return %alloc : memref<1024x8192xf32>
}
+
+// -----
+
+// The external call receives a fully aliasing cast of the producer's memref.
+// Fusion must preserve the call between the producer and consumer loops.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_external_call
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: call @escape
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @cast_alias_external_call(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.cast %comm : memref<32xf64> to memref<?xf64>
+ func.call @escape(%view) : (memref<?xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape(memref<?xf64>)
+
+// -----
+
+// A ranked source cast to an unranked memref must retain the dependence on the
+// source used by the affine producer and consumer.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_ranked_to_unranked
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: call @escape_unranked
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+func.func @cast_alias_ranked_to_unranked(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.cast %comm : memref<32xf64> to memref<*xf64>
+ func.call @escape_unranked(%view) : (memref<*xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape_unranked(memref<*xf64>)
+
+// -----
+
+// An unranked source cast to a ranked memref must retain the dependence on the
+// source passed to the opaque call.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_unranked_to_ranked
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+// PRODUCER-CONSUMER-MAXIMAL: call @escape_ranked
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+func.func @cast_alias_unranked_to_ranked(
+ %in: memref<32xf64>, %comm: memref<*xf64>, %out: memref<32xf64>) {
+ %view = memref.cast %comm : memref<*xf64> to memref<32xf64>
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %view[%i] : memref<32xf64>
+ }
+ func.call @escape_ranked(%comm) : (memref<*xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %view[%j] : memref<32xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape_ranked(memref<*xf64>)
+
+// -----
+
+// Affine accesses may use the fully aliasing view while the external call uses
+// the source value. The call must remain between the two loop nests.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_reverse
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: call @escape_static
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @cast_alias_reverse(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %view = memref.cast %comm : memref<32xf64> to memref<?xf64>
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %view[%i] : memref<?xf64>
+ }
+ func.call @escape_static(%comm) : (memref<32xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %view[%j] : memref<?xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape_static(memref<32xf64>)
+
+// -----
+
+// Two distinct fully aliasing views must be placed in the same dependence
+// class as their source value.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_sibling
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: call @escape
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @cast_alias_sibling(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %view0 = memref.cast %comm : memref<32xf64> to memref<?xf64>
+ %view1 = memref.cast %comm : memref<32xf64> to memref<?xf64>
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %view0[%i] : memref<?xf64>
+ }
+ func.call @escape(%view1) : (memref<?xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %view0[%j] : memref<?xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape(memref<?xf64>)
+
+// -----
+
+// A representable store through a cast alias must retain the producer-consumer
+// dependence without relying on an opaque external call.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_memref_store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.cast
+// PRODUCER-CONSUMER-MAXIMAL: memref.store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @cast_alias_memref_store(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %c0 = arith.constant 0 : index
+ %cst = arith.constant 1.0 : f64
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.cast %comm : memref<32xf64> to memref<?xf64>
+ memref.store %cst, %view[%c0] : memref<?xf64>
+ affine.for %j = 0 to 8 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ affine.store %c, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// The same representable effect on unrelated storage must not block fusion.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @cast_alias_unrelated_memref_store
+// PRODUCER-CONSUMER-MAXIMAL: memref.store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: return
+func.func @cast_alias_unrelated_memref_store(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %other = memref.alloc() : memref<32xf64>
+ %c0 = arith.constant 0 : index
+ %cst = arith.constant 1.0 : f64
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.cast %other : memref<32xf64> to memref<?xf64>
+ memref.store %cst, %view[%c0] : memref<?xf64>
+ affine.for %j = 0 to 8 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ affine.store %c, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// -----
+
+// A private replacement of view A must not remove the exact SSA dependence on
+// sibling view B. The second fusion opportunity is deliberately separated by
+// a use of the value produced by B; dropping that edge would move the fused
+// destination before B's definition.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @incremental_ssa_view_edge_preserved
+// PRODUCER-CONSUMER-MAXIMAL: %[[B:.*]] = memref.subview
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load %{{.*}}[%{{.*}}] : memref<32xf32>
+// PRODUCER-CONSUMER-MAXIMAL: memref.dim %[[B]]
+func.func @incremental_ssa_view_edge_preserved(
+ %in: memref<32xf32>, %out: memref<8xf32>) {
+ %base = memref.alloc() : memref<32xf32>
+ %a = memref.cast %base : memref<32xf32> to memref<?xf32>
+ %y = memref.alloc() : memref<16xf32>
+ %dout = memref.alloc() : memref<16xf32>
+ affine.for %py = 0 to 16 {
+ %v = affine.load %in[%py] : memref<32xf32>
+ affine.store %v, %y[%py] : memref<16xf32>
+ }
+ affine.for %d = 0 to 16 {
+ %v = affine.load %y[%d] : memref<16xf32>
+ affine.store %v, %dout[%d] : memref<16xf32>
+ }
+ affine.for %pa = 0 to 16 {
+ %v = affine.load %in[%pa] : memref<32xf32>
+ affine.store %v, %a[%pa] : memref<?xf32>
+ }
+ %b = memref.subview %base[0] [32] [1]
+ : memref<32xf32> to memref<32xf32, strided<[1], offset: 0>>
+ %c0 = arith.constant 0 : index
+ affine.for %c = 0 to 8 {
+ %dim = memref.dim %b, %c0
+ : memref<32xf32, strided<[1], offset: 0>>
+ %av = affine.load %a[%c] : memref<?xf32>
+ %yv = affine.load %y[%c] : memref<16xf32>
+ %sum = arith.addf %av, %yv : f32
+ affine.store %sum, %out[%c] : memref<8xf32>
+ }
+ return
+}
+
+// -----
+
+// If private-buffer creation fails for multiple producer stores with different
+// access functions, the original graph edges must remain available to the next
+// fusion attempt.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @failed_private_memref_keeps_ssa_edge
+// PRODUCER-CONSUMER-MAXIMAL: %[[FAILED_B:.*]] = memref.subview
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load %{{.*}}[%{{.*}}] : memref<32xf32>
+// PRODUCER-CONSUMER-MAXIMAL: memref.dim %[[FAILED_B]]
+func.func @failed_private_memref_keeps_ssa_edge(
+ %in: memref<32xf32>, %out: memref<8xf32>) {
+ %base = memref.alloc() : memref<32xf32>
+ %a = memref.cast %base : memref<32xf32> to memref<?xf32>
+ %y = memref.alloc() : memref<16xf32>
+ %dout = memref.alloc() : memref<16xf32>
+ affine.for %py = 0 to 16 {
+ %v = affine.load %in[%py] : memref<32xf32>
+ affine.store %v, %y[%py] : memref<16xf32>
+ }
+ affine.for %d = 0 to 16 {
+ %v = affine.load %y[%d] : memref<16xf32>
+ affine.store %v, %dout[%d] : memref<16xf32>
+ }
+ affine.for %pa = 0 to 16 {
+ %v = affine.load %in[%pa] : memref<32xf32>
+ affine.store %v, %a[%pa] : memref<?xf32>
+ affine.store %v, %a[%pa + 1] : memref<?xf32>
+ }
+ %b = memref.subview %base[0] [32] [1]
+ : memref<32xf32> to memref<32xf32, strided<[1], offset: 0>>
+ %c0 = arith.constant 0 : index
+ affine.for %c = 0 to 8 {
+ %dim = memref.dim %b, %c0
+ : memref<32xf32, strided<[1], offset: 0>>
+ %av = affine.load %a[%c] : memref<?xf32>
+ %yv = affine.load %y[%c] : memref<16xf32>
+ %sum = arith.addf %av, %yv : f32
+ affine.store %sum, %out[%c] : memref<8xf32>
+ }
+ return
+}
+
+// -----
+
+// A zero-offset, unit-stride subview is a fully aliasing view and must retain
+// the same dependence as its source memref.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @subview_alias_external_call
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+// PRODUCER-CONSUMER-MAXIMAL: memref.subview
+// PRODUCER-CONSUMER-MAXIMAL: call @escape_subview
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+func.func @subview_alias_external_call(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.subview %comm[0] [32] [1]
+ : memref<32xf64> to memref<32xf64, strided<[1], offset: 0>>
+ func.call @escape_subview(%view)
+ : (memref<32xf64, strided<[1], offset: 0>>) -> ()
+ affine.for %j = 0 to 16 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ %d = arith.addf %c, %c : f64
+ affine.store %d, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @escape_subview(memref<32xf64, strided<[1], offset: 0>>)
+
+// -----
+
+// A non-zero-offset subview of the producer's storage must retain a
+// dependence even though its affine coordinates use a different view.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @subview_overlap_memref_store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.subview
+// PRODUCER-CONSUMER-MAXIMAL: memref.store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @subview_overlap_memref_store(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %c0 = arith.constant 0 : index
+ %cst = arith.constant 1.0 : f64
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %comm[%i] : memref<32xf64>
+ }
+ %view = memref.subview %comm[1] [16] [1]
+ : memref<32xf64> to memref<16xf64, strided<[1], offset: 1>>
+ memref.store %cst, %view[%c0]
+ : memref<16xf64, strided<[1], offset: 1>>
+ affine.for %j = 0 to 16 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ affine.store %c, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// -----
+
+// A non-zero-offset subview of unrelated storage must not block an otherwise
+// legal producer-consumer fusion.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @subview_non_alias_memref_store
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.store
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+// PRODUCER-CONSUMER-MAXIMAL: return
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+func.func @subview_non_alias_memref_store(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ %other = memref.alloc() : memref<32xf64>
+ %view = memref.subview %other[1] [16] [1]
+ : memref<32xf64> to memref<16xf64, strided<[1], offset: 1>>
+ %c0 = arith.constant 0 : index
+ %cst = arith.constant 1.0 : f64
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ %b = arith.addf %a, %a : f64
+ affine.store %b, %comm[%i] : memref<32xf64>
+ }
+ memref.store %cst, %view[%c0]
+ : memref<16xf64, strided<[1], offset: 1>>
+ affine.for %j = 0 to 16 {
+ %c = affine.load %comm[%j] : memref<32xf64>
+ affine.store %c, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// -----
+
+// A represented free must remain a dependence through full affine filtering.
+// Otherwise fusing the first and third loops could move a store to %a past its
+// deallocation.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @representable_free_between_loops
+// PRODUCER-CONSUMER-MAXIMAL: %[[A:.*]] = memref.alloc
+// PRODUCER-CONSUMER-MAXIMAL: affine.store {{.*}}, %[[A]][
+// PRODUCER-CONSUMER-MAXIMAL: memref.dealloc %[[A]]
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.store {{.*}}, %[[A]][
+// PRODUCER-CONSUMER-MAXIMAL: return
+func.func @representable_free_between_loops(
+ %in: memref<32xf64>, %b: memref<32xf64>, %out: memref<32xf64>) {
+ %a = memref.alloc() : memref<32xf64>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %in[%i] : memref<32xf64>
+ affine.store %v, %a[%i] : memref<32xf64>
+ affine.store %v, %b[%i] : memref<32xf64>
+ }
+ affine.for %k = 0 to 1 {
+ memref.dealloc %a : memref<32xf64>
+ }
+ affine.for %j = 0 to 16 {
+ %v = affine.load %b[%j] : memref<32xf64>
+ affine.store %v, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// -----
+
+// An unknown call without memref operands can access a global memref and must
+// not become an isolated node that fusion can cross.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @unknown_call_global_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: call @touch_global
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @unknown_call_global_effect(
+ %in: memref<32xf64>, %out: memref<32xf64>) {
+ %global = memref.get_global @fusion_global : memref<32xf64>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %in[%i] : memref<32xf64>
+ affine.store %v, %global[%i] : memref<32xf64>
+ }
+ func.call @touch_global() : () -> ()
+ affine.for %j = 0 to 16 {
+ %v = affine.load %global[%j] : memref<32xf64>
+ affine.store %v, %out[%j] : memref<32xf64>
+ }
+ return
+}
+memref.global "private" @fusion_global : memref<32xf64>
+func.func private @touch_global()
+
+// An unknown non-call operation can access memory not represented by its
+// operands and must not become an isolated node that fusion can cross.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @unknown_non_call_without_operands_prevents_fusion
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: }
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: "unknown.touch_global"() : () -> ()
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+func.func @unknown_non_call_without_operands_prevents_fusion(
+ %in: memref<32xf64>, %out: memref<32xf64>) {
+ %global = memref.get_global @fusion_global : memref<32xf64>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %in[%i] : memref<32xf64>
+ affine.store %v, %global[%i] : memref<32xf64>
+ }
+ "unknown.touch_global"() : () -> ()
+ affine.for %j = 0 to 16 {
+ %v = affine.load %global[%j] : memref<32xf64>
+ affine.store %v, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// Explicit operands do not make the effects of an unknown operation complete.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @unknown_non_call_with_memref_operand_may_have_implicit_effects
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: }
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: "unknown.touch_global"(%{{.*}}) : (memref<32xf64>) -> ()
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+func.func @unknown_non_call_with_memref_operand_may_have_implicit_effects(
+ %in: memref<32xf64>, %unrelated: memref<32xf64>, %out: memref<32xf64>) {
+ %global = memref.get_global @fusion_global : memref<32xf64>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %in[%i] : memref<32xf64>
+ affine.store %v, %global[%i] : memref<32xf64>
+ }
+ "unknown.touch_global"(%unrelated) : (memref<32xf64>) -> ()
+ affine.for %j = 0 to 16 {
+ %v = affine.load %global[%j] : memref<32xf64>
+ affine.store %v, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// -----
+
+// An addressable effect without an SSA memory value cannot be represented by
+// the per-memref dependence graph. Fusion must be skipped for the block.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @value_less_addressable_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @value_less_addressable_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ %effect = "test.side_effect_op"() {effects = [{effect = "write"}]} : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// A symbol-associated addressable effect is also not representable by a
+// per-memref dependence graph. Fusion must be skipped for the block.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @symbol_addressable_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @symbol_addressable_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ "test.side_effect_op"() {
+ effects = [{effect = "write", on_reference = @effect_target}]
+ } : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ }
+ return
+}
+func.func private @effect_target()
+
+// An addressable effect on a non-memref SSA value is not representable by the
+// per-memref dependence graph. Fusion must be skipped for the block.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @value_addressable_nonmemref_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @value_addressable_nonmemref_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ %effect = "test.side_effect_op"() {
+ effects = [{effect = "write", on_result}]
+ } : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// A value-less effect on a non-addressable resource cannot be represented by
+// the per-memref dependence graph. Effects on the same resource must still be
+// ordered, so fusion must be skipped for the block.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @value_less_nonaddressable_write_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @value_less_nonaddressable_write_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ "test.side_effect_op"() {
+ effects = [{effect = "write", test_nonaddressable_resource}]
+ } : () -> i32
+ }
+ "test.side_effect_op"() {
+ effects = [{effect = "read", test_nonaddressable_resource}]
+ } : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// Value-less reads on a non-addressable resource are also unrepresentable.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @value_less_nonaddressable_read_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+func.func @value_less_nonaddressable_read_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ }
+ "test.side_effect_op"() {
+ effects = [{effect = "read", test_nonaddressable_resource}]
+ } : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ "test.side_effect_op"() {
+ effects = [{effect = "write", test_nonaddressable_resource}]
+ } : () -> i32
+ }
+ return
+}
+
+// Free effects follow the same conservative rule.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @value_less_nonaddressable_free_effect
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @value_less_nonaddressable_free_effect(
+ %in: memref<32xf64>, %comm: memref<32xf64>, %out: memref<32xf64>) {
+ affine.for %i = 0 to 16 {
+ %a = affine.load %in[%i] : memref<32xf64>
+ affine.store %a, %comm[%i] : memref<32xf64>
+ "test.side_effect_op"() {
+ effects = [{effect = "free", test_nonaddressable_resource}]
+ } : () -> i32
+ }
+ "test.side_effect_op"() {
+ effects = [{effect = "read", test_nonaddressable_resource}]
+ } : () -> i32
+ affine.for %j = 0 to 16 {
+ %a = affine.load %comm[%j] : memref<32xf64>
+ affine.store %a, %out[%j] : memref<32xf64>
+ }
+ return
+}
+
+// Non-affine memory effects inside candidate loops are not included in the
+// affine slice analysis, so fusion must preserve the two complete loops.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @non_affine_effects_inside_candidate_loops
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.load
+// PRODUCER-CONSUMER-MAXIMAL: memref.store
+// PRODUCER-CONSUMER-MAXIMAL: }
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: memref.load
+func.func @non_affine_effects_inside_candidate_loops(
+ %in: memref<8xi32>, %out: memref<8xi32>) {
+ %tmp = memref.alloc() : memref<8xi32>
+ %counter = memref.alloc() : memref<1xi32>
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : i32
+ affine.for %i = 0 to 8 {
+ %v = affine.load %in[%i] : memref<8xi32>
+ %old = memref.load %counter[%c0] : memref<1xi32>
+ %next = arith.addi %old, %c1 : i32
+ memref.store %next, %counter[%c0] : memref<1xi32>
+ affine.store %v, %tmp[%i] : memref<8xi32>
+ }
+ affine.for %j = 0 to 8 {
+ %v = affine.load %tmp[%j] : memref<8xi32>
+ %current = memref.load %counter[%c0] : memref<1xi32>
+ %result = arith.addi %v, %current : i32
+ affine.store %result, %out[%j] : memref<8xi32>
+ }
+ return
+}
+
+// Allocation is an effect even when it has no memref value. Slicing the source
+// loop must not change the number or order of such effects.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @allocation_effect_inside_candidate_loop
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: test.side_effect_op
+// PRODUCER-CONSUMER-MAXIMAL-NOT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+func.func @allocation_effect_inside_candidate_loop(
+ %in: memref<16xi32>, %out: memref<16xi32>) {
+ %tmp = memref.alloc() : memref<16xi32>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %in[%i] : memref<16xi32>
+ "test.side_effect_op"() {effects = [{effect = "allocate"}]} : () -> i32
+ affine.store %v, %tmp[%i] : memref<16xi32>
+ }
+ affine.for %j = 0 to 8 {
+ %v = affine.load %tmp[%j] : memref<16xi32>
+ affine.store %v, %out[%j] : memref<16xi32>
+ }
+ return
+}
+
+// A source read and destination write through overlapping non-trivial views
+// must not be interleaved by producer-consumer fusion.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @overlapping_subviews_inside_loops
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+// PRODUCER-CONSUMER-MAXIMAL: }
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+func.func @overlapping_subviews_inside_loops(%root: memref<32xf32>) {
+ %a = memref.subview %root[0] [16] [1]
+ : memref<32xf32> to memref<16xf32, strided<[1], offset: 0>>
+ %b = memref.subview %root[1] [16] [1]
+ : memref<32xf32> to memref<16xf32, strided<[1], offset: 1>>
+ %tmp = memref.alloc() : memref<16xf32>
+ affine.for %i = 0 to 16 {
+ %v = affine.load %a[%i]
+ : memref<16xf32, strided<[1], offset: 0>>
+ affine.store %v, %tmp[%i] : memref<16xf32>
+ }
+ affine.for %j = 0 to 16 {
+ %v = affine.load %tmp[%j] : memref<16xf32>
+ affine.store %v, %b[%j]
+ : memref<16xf32, strided<[1], offset: 1>>
+ }
+ return
+}
+
+// The reverse read/write direction is covered independently so the guard is
+// not accidentally limited to source-read/destination-write pairs.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @overlapping_subviews_write_read
+// PRODUCER-CONSUMER-MAXIMAL: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.store
+// PRODUCER-CONSUMER-MAXIMAL: }
+// PRODUCER-CONSUMER-MAXIMAL-NEXT: affine.for
+// PRODUCER-CONSUMER-MAXIMAL: affine.load
+func.func @overlapping_subviews_write_read(
+ %root: memref<32xf32>, %out: memref<16xf32>) {
+ %a = memref.subview %root[0] [16] [1]
+ : memref<32xf32> to memref<16xf32, strided<[1], offset: 0>>
+ %b = memref.subview %root[1] [16] [1]
+ : memref<32xf32> to memref<16xf32, strided<[1], offset: 1>>
+ %tmp = memref.alloc() : memref<16xf32>
+ %cst = arith.constant 1.0 : f32
+ affine.for %i = 0 to 16 {
+ affine.store %cst, %a[%i]
+ : memref<16xf32, strided<[1], offset: 0>>
+ affine.store %cst, %tmp[%i] : memref<16xf32>
+ }
+ affine.for %j = 0 to 16 {
+ %v = affine.load %tmp[%j] : memref<16xf32>
+ %other = affine.load %b[%j]
+ : memref<16xf32, strided<[1], offset: 1>>
+ %sum = arith.addf %v, %other : f32
+ affine.store %sum, %out[%j] : memref<16xf32>
+ }
+ return
+}
+
+// -----
+
+// The destination nest has an affine loop-carried dependence that makes its
+// outer loop sequential. The non-affine recurrence is not part of the affine
+// dependence components, so sinking the sequential loop would change its
+// schedule even though fusion is later rejected.
+
+#linear = affine_map<(d0, d1) -> (d0 * 2 + d1)>
+
+// NO-REORDER-LABEL: func.func @non_affine_effect_prevents_loop_sinking(
+// NO-REORDER: affine.for %{{.*}} = 0 to 5 {
+// NO-REORDER-NEXT: affine.for %{{.*}} = 0 to 3 {
+// NO-REORDER: affine.for %{{.*}} = 0 to 4 {
+// NO-REORDER-NEXT: affine.for %{{.*}} = 0 to 3 {
+func.func @non_affine_effect_prevents_loop_sinking() {
+ %dep = memref.alloc() : memref<5x3xi64>
+ %state = memref.alloc() : memref<1xi64>
+ %out = memref.alloc() : memref<4x3xi64>
+ %c0 = arith.constant 0 : index
+ %zero = arith.constant 0 : i64
+ %ten = arith.constant 10 : i64
+ memref.store %zero, %state[%c0] : memref<1xi64>
+
+ affine.for %p = 0 to 5 {
+ affine.for %q = 0 to 3 {
+ affine.store %zero, %dep[%p, %q] : memref<5x3xi64>
+ }
+ }
+ affine.for %i = 0 to 4 {
+ affine.for %j = 0 to 3 {
+ %v = affine.load %dep[%i, %j] : memref<5x3xi64>
+ affine.store %v, %dep[%i + 1, %j] : memref<5x3xi64>
+ %linear = affine.apply #linear(%i, %j)
+ %linear_i64 = arith.index_cast %linear : index to i64
+ %old = memref.load %state[%c0] : memref<1xi64>
+ %scaled = arith.muli %old, %ten : i64
+ %next = arith.addi %scaled, %linear_i64 : i64
+ memref.store %next, %state[%c0] : memref<1xi64>
+ affine.store %v, %out[%i, %j] : memref<4x3xi64>
+ }
+ }
+ return
+}
+
+// Affine accesses through different views of the same storage are not
+// represented by the raw-view dependence components. They must still prevent
+// sinking when another affine dependence would otherwise select interchange.
+
+// NO-REORDER-LABEL: func.func @affine_alias_prevents_loop_sinking(
+// NO-REORDER: affine.for %{{.*}} = 0 to 1 {
+// NO-REORDER: affine.store
+// NO-REORDER: affine.for %{{.*}} = 1 to 5 {
+// NO-REORDER-NEXT: affine.for %{{.*}} = 1 to 4 {
+func.func @affine_alias_prevents_loop_sinking(
+ %seed: memref<1xi64>, %dep: memref<6x4xi64>, %root: memref<5x5xi64>) {
+ %view = memref.cast %root : memref<5x5xi64> to memref<?x?xi64>
+ %zero = arith.constant 0 : i64
+ affine.for %s = 0 to 1 {
+ affine.store %zero, %seed[%s] : memref<1xi64>
+ }
+ affine.for %i = 1 to 5 {
+ affine.for %j = 1 to 4 {
+ %v = affine.load %dep[%i, %j] : memref<6x4xi64>
+ affine.store %v, %dep[%i + 1, %j] : memref<6x4xi64>
+ %old = affine.load %root[%i - 1, %j + 1] : memref<5x5xi64>
+ %next = arith.addi %old, %v : i64
+ affine.store %next, %view[%i, %j] : memref<?x?xi64>
+ }
+ }
+ return
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir b/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir
index 2c53852a8cec9..cd4a798ebf088 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir
@@ -1,4 +1,5 @@
// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion=test-loop-fusion-dependence-check -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion=test-loop-fusion-utilities -split-input-file | FileCheck %s --check-prefix=UTILITY
// -----
@@ -109,7 +110,6 @@ func.func @should_not_fuse_across_intermediate_store() {
affine.for %i0 = 0 to 10 {
// expected-remark at -1 {{block-level dependence preventing fusion of loop nest 0 into loop nest 1 at depth 0}}
%v0 = affine.load %0[%i0] : memref<10xf32>
- "op0"(%v0) : (f32) -> ()
}
// Should not fuse loop nests '%i0' and '%i1' across top-level store.
@@ -118,7 +118,6 @@ func.func @should_not_fuse_across_intermediate_store() {
affine.for %i1 = 0 to 10 {
// expected-remark at -1 {{block-level dependence preventing fusion of loop nest 1 into loop nest 0 at depth 0}}
%v1 = affine.load %0[%i1] : memref<10xf32>
- "op1"(%v1) : (f32) -> ()
}
return
}
@@ -138,7 +137,6 @@ func.func @should_not_fuse_across_intermediate_load() {
// Should not fuse loop nests '%i0' and '%i1' across top-level load.
%v0 = affine.load %0[%c0] : memref<10xf32>
- "op0"(%v0) : (f32) -> ()
affine.for %i1 = 0 to 10 {
// expected-remark at -1 {{block-level dependence preventing fusion of loop nest 1 into loop nest 0 at depth 0}}
@@ -165,7 +163,6 @@ func.func @should_not_fuse_across_ssa_value_def() {
// Loop nest '%i0" cannot be fused past load from '%1' due to RAW dependence.
%v1 = affine.load %1[%c0] : memref<10xf32>
- "op0"(%v1) : (f32) -> ()
// Loop nest '%i1' cannot be fused past SSA value def '%c2' which it uses.
%c2 = arith.constant 2 : index
@@ -306,6 +303,58 @@ func.func @should_not_fuse_across_store_in_loop_at_depth1() {
// -----
+// The public utility must reject an unknown effect even when no MDG was built
+// by the caller first.
+
+// UTILITY-LABEL: func.func @utility_rejects_unknown_effect_in_candidate_loop
+// UTILITY: affine.for
+// UTILITY: "unknown.effect"() : () -> ()
+// UTILITY-NEXT: affine.store
+// UTILITY-NEXT: }
+// UTILITY-NEXT: affine.for
+func.func @utility_rejects_unknown_effect_in_candidate_loop(
+ %in: memref<8xi32>, %out: memref<8xi32>) {
+ %tmp = memref.alloc() : memref<8xi32>
+ affine.for %i = 0 to 8 {
+ %v = affine.load %in[%i] : memref<8xi32>
+ "unknown.effect"() : () -> ()
+ affine.store %v, %tmp[%i] : memref<8xi32>
+ }
+ affine.for %j = 0 to 8 {
+ %v = affine.load %tmp[%j] : memref<8xi32>
+ affine.store %v, %out[%j] : memref<8xi32>
+ }
+ return
+}
+
+// -----
+
+// Allocation effects are not safe to clone into a sliced loop body, even when
+// the effect has no associated memref value.
+
+// UTILITY-LABEL: func.func @utility_rejects_allocate_effect_in_candidate_loop
+// UTILITY: affine.for
+// UTILITY: "test.side_effect_op"
+// UTILITY-NEXT: affine.store
+// UTILITY-NEXT: }
+// UTILITY-NEXT: affine.for
+func.func @utility_rejects_allocate_effect_in_candidate_loop(
+ %in: memref<8xi32>, %out: memref<8xi32>) {
+ %tmp = memref.alloc() : memref<8xi32>
+ affine.for %i = 0 to 8 {
+ %v = affine.load %in[%i] : memref<8xi32>
+ "test.side_effect_op"() {effects = [{effect = "allocate"}]} : () -> i32
+ affine.store %v, %tmp[%i] : memref<8xi32>
+ }
+ affine.for %j = 0 to 8 {
+ %v = affine.load %tmp[%j] : memref<8xi32>
+ affine.store %v, %out[%j] : memref<8xi32>
+ }
+ return
+}
+
+// -----
+
// CHECK-LABEL: func @should_not_fuse_across_ssa_value_def_at_depth1() {
func.func @should_not_fuse_across_ssa_value_def_at_depth1() {
%0 = memref.alloc() : memref<10x10xf32>
@@ -323,7 +372,6 @@ func.func @should_not_fuse_across_ssa_value_def_at_depth1() {
// RAW dependence from store in loop nest '%i1' to 'load %1' prevents
// fusion loop nest '%i1' into loops after load.
%v1 = affine.load %1[%i0, %c0] : memref<10x10xf32>
- "op0"(%v1) : (f32) -> ()
// Loop nest '%i2' cannot be fused past SSA value def '%c2' which it uses.
%c2 = arith.constant 2 : index
@@ -334,4 +382,4 @@ func.func @should_not_fuse_across_ssa_value_def_at_depth1() {
}
}
return
-}
\ No newline at end of file
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-edge-kinds.mlir b/mlir/test/Dialect/Affine/loop-fusion-edge-kinds.mlir
new file mode 100644
index 0000000000000..2c8aa8bf5181d
--- /dev/null
+++ b/mlir/test/Dialect/Affine/loop-fusion-edge-kinds.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion=test-loop-fusion-edge-kinds | FileCheck %s
+
+// CHECK: edge-kinds ssa-only any=1 memory=0 ssa=1 out=1 in=1 count=1
+// CHECK: edge-kinds both any=1 memory=1 ssa=1 out=2 in=2 count=2
+// CHECK: edge-kinds ssa-after-memory-removal any=1 memory=0 ssa=1 out=1 in=1 count=1
+// CHECK: edge-kinds empty any=0 memory=0 ssa=0 out=0 in=0 count=0
+func.func @edge_kinds() {
+ %src = memref.alloc() : memref<1xi64>
+ %dst = memref.alloc() : memref<1xi64>
+ return
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-failure.mlir b/mlir/test/Dialect/Affine/loop-fusion-failure.mlir
new file mode 100644
index 0000000000000..2755b34dbb079
--- /dev/null
+++ b/mlir/test/Dialect/Affine/loop-fusion-failure.mlir
@@ -0,0 +1,30 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion=test-loop-fusion-failure -split-input-file -verify-diagnostics | FileCheck %s
+
+#dynamic_index = affine_map<()[s0, s1] -> (s0 * s1)>
+
+// An unsupported same-memref access relation must stop producer-consumer
+// fusion at the dependence check. The destination loops remain separate.
+
+// CHECK-LABEL: func.func @failed_dependence_fusion(
+// CHECK: affine.for %{{.*}} = 1 to 8
+// CHECK: affine.for %{{.*}} = 1 to 8
+// CHECK: affine.for %{{.*}} = 1 to 8
+// CHECK: affine.for %{{.*}} = 1 to 8
+func.func @failed_dependence_fusion(
+ %A: memref<?x9x9xi32>, %p: index, %q: index, %value: i32) {
+ affine.for %i = 1 to 8 {
+ // expected-remark at -1 {{fusion dependence prevents fusion at depth 1}}
+ affine.for %j = 1 to 8 {
+ %z = affine.apply #dynamic_index()[%p, %q]
+ affine.store %value, %A[%z, %i, %j] : memref<?x9x9xi32>
+ }
+ }
+ affine.for %i = 1 to 8 {
+ affine.for %j = 1 to 8 {
+ %z = affine.apply #dynamic_index()[%p, %q]
+ %loaded = affine.load %A[%z, %i - 1, %j + 1] : memref<?x9x9xi32>
+ affine.store %loaded, %A[%z, %i - 1, %j + 1] : memref<?x9x9xi32>
+ }
+ }
+ return
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-utilities.mlir b/mlir/test/Dialect/Affine/loop-fusion-utilities.mlir
index 11435ad2d0203..416d155d763fe 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-utilities.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-utilities.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -allow-unregistered-dialect -test-loop-fusion=test-loop-fusion-utilities -split-input-file -canonicalize | FileCheck %s
+// RUN: mlir-opt %s -allow-unregistered-dialect -test-loop-fusion=test-loop-fusion-utilities -split-input-file | FileCheck %s
// CHECK-LABEL: func @slice_depth1_loop_nest() {
func.func @slice_depth1_loop_nest() {
@@ -9,14 +9,13 @@ func.func @slice_depth1_loop_nest() {
}
affine.for %i1 = 0 to 5 {
%1 = affine.load %0[%i1] : memref<100xf32>
- "prevent.dce"(%1) : (f32) -> ()
+ "test.side_effect_op"() : () -> i32
}
// CHECK: affine.for %[[IV0:.*]] = 0 to 5 {
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%[[IV0]]] : memref<100xf32>
// CHECK-NEXT: affine.load %{{.*}}[%[[IV0]]] : memref<100xf32>
- // CHECK-NEXT: "prevent.dce"(%{{.*}}) : (f32) -> ()
- // CHECK-NEXT: }
- // CHECK-NEXT: return
+ // CHECK: }
+ // CHECK: return
return
}
@@ -81,7 +80,7 @@ func.func @should_fuse_avoiding_dependence_cycle() {
affine.for %i1 = 0 to 10 {
affine.store %cf7, %a[%i1] : memref<10xf32>
%v1 = affine.load %c[%i1] : memref<10xf32>
- "prevent.dce"(%v1) : (f32) -> ()
+ "test.side_effect_op"() : () -> i32
}
affine.for %i2 = 0 to 10 {
%v2 = affine.load %b[%i2] : memref<10xf32>
@@ -95,15 +94,17 @@ func.func @should_fuse_avoiding_dependence_cycle() {
// Then fuse this loop nest with loop2:
// {0, 1, 2}
//
+ // CHECK-NOT: affine.for
// CHECK: affine.for %{{.*}} = 0 to 10 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
- // CHECK-NEXT: "prevent.dce"
- // CHECK-NEXT: affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+ // CHECK-NOT: affine.for
+ // CHECK: affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
// CHECK-NEXT: }
+ // CHECK-NOT: affine.for
// CHECK-NEXT: return
return
}
diff --git a/mlir/test/Dialect/Affine/loop-fusion.mlir b/mlir/test/Dialect/Affine/loop-fusion.mlir
index 1ea42517988c3..b55eee5c27b6e 100644
--- a/mlir/test/Dialect/Affine/loop-fusion.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion.mlir
@@ -596,7 +596,7 @@ func.func @permute_and_fuse() {
affine.for %i4 = 0 to 10 {
affine.for %i5 = 0 to 20 {
%v0 = affine.load %m[%i4, %i5, %i3] : memref<10x20x30xf32>
- "foo"(%v0) : (f32) -> ()
+ %unused = arith.addf %v0, %v0 : f32
}
}
}
@@ -605,7 +605,7 @@ func.func @permute_and_fuse() {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 20 {
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0, 0, 0] : memref<1x1x1xf32>
// CHECK-NEXT: affine.load %{{.*}}[0, 0, 0] : memref<1x1x1xf32>
-// CHECK-NEXT: "foo"(%{{.*}}) : (f32) -> ()
+// CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: }
// CHECK-NEXT: }
@@ -633,7 +633,7 @@ func.func @fuse_reshape_64_16_4(%in : memref<64xf32>) {
affine.for %i1 = 0 to 16 {
affine.for %i2 = 0 to 4 {
%w = affine.load %out[%i1, %i2] : memref<16x4xf32>
- "foo"(%w) : (f32) -> ()
+ %unused = arith.addf %w, %w : f32
}
}
return
@@ -665,7 +665,7 @@ func.func @fuse_reshape_16_4_64() {
affine.for %i2 = 0 to 64 {
%w = affine.load %out[%i2] : memref<64xf32>
- "foo"(%w) : (f32) -> ()
+ %unused = arith.addf %w, %w : f32
}
// CHECK: affine.for %{{.*}} = 0 to 64 {
// CHECK-NEXT: affine.apply [[$MAP0]](%{{.*}})
@@ -674,7 +674,7 @@ func.func @fuse_reshape_16_4_64() {
// CHECK-NEXT: affine.apply [[$MAP2]](%{{.*}}, %{{.*}})
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
// CHECK-NEXT: affine.load %{{.*}}[0] : memref<1xf32>
-// CHECK-NEXT: "foo"(%{{.*}}) : (f32) -> ()
+// CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: return
return
@@ -697,7 +697,7 @@ func.func @R6_to_R2_reshape_square() -> memref<64x9xi32> {
affine.for %i3 = 0 to 3 {
affine.for %i4 = 0 to 16 {
affine.for %i5 = 0 to 1 {
- %val = "foo"(%i0, %i1, %i2, %i3, %i4, %i5) : (index, index, index, index, index, index) -> i32
+ %val = arith.constant 0 : i32
affine.store %val, %in[%i0, %i1, %i2, %i3, %i4, %i5] : memref<2x2x3x3x16x1xi32>
}
}
@@ -758,7 +758,7 @@ func.func @R6_to_R2_reshape_square() -> memref<64x9xi32> {
// CHECK-NEXT: affine.apply [[$MAP2]](%{{.*}}, %{{.*}})
// CHECK-NEXT: affine.apply [[$MAP3]](%{{.*}}, %{{.*}})
// CHECK-NEXT: affine.apply [[$MAP4]](%{{.*}}, %{{.*}})
-// CHECK-NEXT: "foo"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (index, index, index, index, index, index) -> i32
+// CHECK-NEXT: %{{.*}} = arith.constant {{.*}} : i32
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0, 0, 0, 0, 0, 0] : memref<1x1x1x1x1x1xi32>
// CHECK-NEXT: affine.apply [[$MAP11]](%{{.*}}, %{{.*}})
// CHECK-NEXT: affine.apply [[$MAP12]](%{{.*}})
@@ -812,7 +812,7 @@ func.func @should_fuse_reduction_at_depth_of_one() {
affine.for %i1 = 0 to 100 {
%v0 = affine.load %b[%i0] : memref<10xf32>
%v1 = affine.load %a[%i0, %i1] : memref<10x100xf32>
- %v2 = "maxf"(%v0, %v1) : (f32, f32) -> f32
+ %v2 = arith.addf %v0, %v1 : f32
affine.store %v2, %b[%i0] : memref<10xf32>
}
}
@@ -832,7 +832,7 @@ func.func @should_fuse_reduction_at_depth_of_one() {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 100 {
// CHECK-NEXT: affine.load %{{.*}}[0] : memref<1xf32>
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x100xf32>
- // CHECK-NEXT: "maxf"(%{{.*}}, %{{.*}}) : (f32, f32) -> f32
+ // CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0] : memref<1xf32>
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 100 {
@@ -856,10 +856,10 @@ func.func @should_fuse_at_src_depth1_and_dst_depth1() {
affine.for %i0 = 0 to 100 {
affine.for %i1 = 0 to 16 {
%v0 = affine.load %a[%i0, %i1] : memref<100x16xf32>
- "op0"(%v0) : (f32) -> ()
+ %unused0 = arith.addf %v0, %v0 : f32
}
affine.for %i2 = 0 to 16 {
- %v1 = "op1"() : () -> (f32)
+ %v1 = arith.constant 0.0 : f32
affine.store %v1, %b[%i0, %i2] : memref<100x16xf32>
}
}
@@ -867,7 +867,7 @@ func.func @should_fuse_at_src_depth1_and_dst_depth1() {
affine.for %i3 = 0 to 100 {
affine.for %i4 = 0 to 16 {
%v2 = affine.load %b[%i3, %i4] : memref<100x16xf32>
- "op2"(%v2) : (f32) -> ()
+ %unused2 = arith.addf %v2, %v2 : f32
}
}
// We can slice iterations of the '%i0' and '%i1' loops in the source
@@ -878,15 +878,15 @@ func.func @should_fuse_at_src_depth1_and_dst_depth1() {
// CHECK: affine.for %{{.*}} = 0 to 100 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<100x16xf32>
- // CHECK-NEXT: "op0"(%{{.*}}) : (f32) -> ()
+ // CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
- // CHECK-NEXT: %{{.*}} = "op1"() : () -> f32
+ // CHECK-NEXT: %{{.*}} = arith.constant {{.*}} : f32
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0, %{{.*}}] : memref<1x16xf32>
// CHECK-NEXT: }
// CHECK-NEXT: affine.for %{{.*}} = 0 to 16 {
// CHECK-NEXT: affine.load %{{.*}}[0, %{{.*}}] : memref<1x16xf32>
- // CHECK-NEXT: "op2"(%{{.*}}) : (f32) -> ()
+ // CHECK-NEXT: arith.addf %{{.*}}, %{{.*}} : f32
// CHECK-NEXT: }
// CHECK-NEXT: }
// CHECK-NEXT: return
@@ -1304,7 +1304,7 @@ func.func @R3_to_R2_reshape() {
affine.for %i0 = 0 to 2 {
affine.for %i1 = 0 to 3 {
affine.for %i2 = 0 to 16 {
- %val = "foo"(%i0, %i1, %i2) : (index, index, index) -> i32
+ %val = arith.constant 0 : i32
affine.store %val, %in[%i0, %i1, %i2] : memref<2x3x16xi32>
}
}
@@ -1329,7 +1329,7 @@ func.func @R3_to_R2_reshape() {
// CHECK: affine.for %{{.*}} = 0 to 32 {
// CHECK-NEXT: affine.for %{{.*}} = 0 to 3 {
// CHECK-NEXT: affine.apply [[$MAP0]](%{{.*}}, %{{.*}})
-// CHECK-NEXT: "foo"(%{{.*}}, %{{.*}}, %{{.*}}) : (index, index, index) -> i32
+// CHECK-NEXT: %{{.*}} = arith.constant {{.*}} : i32
// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[0, 0, 0] : memref<1x1x1xi32>
// CHECK-NEXT: affine.apply [[$MAP1]](%{{.*}}, %{{.*}})
// CHECK-NEXT: affine.apply [[$MAP2]](%{{.*}})
@@ -1574,5 +1574,60 @@ func.func @producer_consumer_with_outmost_user(%arg0 : f16) {
return
}
-// Add further tests in mlir/test/Transforms/loop-fusion-4.mlir
+// -----
+
+// Unknown operations nested in an affine loop may access memory outside their
+// explicit operands. Fusion must leave the block unchanged when their effects
+// cannot be represented by the memref dependence graph.
+
+// CHECK-LABEL: func @nested_unknown_call
+// CHECK: affine.for
+// CHECK: func.call @escape_nested
+// CHECK: affine.store
+// CHECK: }
+// CHECK-NEXT: affine.for
+// CHECK: affine.load
+// CHECK: return
+func.func @nested_unknown_call(%m: memref<8xf32>, %out: memref<8xf32>) {
+ %tmp = memref.alloc() : memref<8xf32>
+ affine.for %i = 0 to 8 {
+ %v = affine.load %m[%i] : memref<8xf32>
+ func.call @escape_nested(%m) : (memref<8xf32>) -> ()
+ affine.store %v, %tmp[%i] : memref<8xf32>
+ }
+ affine.for %i = 0 to 8 {
+ %v = affine.load %tmp[%i] : memref<8xf32>
+ affine.store %v, %out[%i] : memref<8xf32>
+ }
+ return
+}
+func.func private @escape_nested(memref<8xf32>)
+
+// -----
+
+// Multi-memref operations must follow the same arbitrary-operation path.
+
+// CHECK-LABEL: func @nested_memref_copy
+// CHECK: affine.for
+// CHECK: memref.copy
+// CHECK: affine.store
+// CHECK: }
+// CHECK-NEXT: affine.for
+// CHECK: affine.load
+// CHECK: return
+func.func @nested_memref_copy(
+ %src: memref<8xf32>, %dst: memref<8xf32>, %out: memref<8xf32>) {
+ %tmp = memref.alloc() : memref<8xf32>
+ affine.for %i = 0 to 8 {
+ memref.copy %src, %dst : memref<8xf32> to memref<8xf32>
+ %v = affine.load %src[%i] : memref<8xf32>
+ affine.store %v, %tmp[%i] : memref<8xf32>
+ }
+ affine.for %i = 0 to 8 {
+ %v = affine.load %tmp[%i] : memref<8xf32>
+ affine.store %v, %out[%i] : memref<8xf32>
+ }
+ return
+}
+// Add further tests in mlir/test/Transforms/loop-fusion-4.mlir
diff --git a/mlir/test/lib/Analysis/TestMemRefDependenceCheck.cpp b/mlir/test/lib/Analysis/TestMemRefDependenceCheck.cpp
index b3b9a590773b0..b08e1e7cd1df0 100644
--- a/mlir/test/lib/Analysis/TestMemRefDependenceCheck.cpp
+++ b/mlir/test/lib/Analysis/TestMemRefDependenceCheck.cpp
@@ -89,7 +89,7 @@ static void checkDependences(ArrayRef<Operation *> loadsAndStores) {
if (result.value == DependenceResult::Failure) {
srcOpInst->emitError("dependence check failed");
} else {
- bool ret = hasDependence(result);
+ bool ret = mustHaveDependence(result);
// TODO: Print dependence type (i.e. RAW, etc) and print
// distance vectors as: ([2, 3], [0, 10]). Also, shorten distance
// vectors from ([1, 1], [3, 3]) to (1, 3).
diff --git a/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp b/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp
index bf11d94596fa7..ca22e6129787d 100644
--- a/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp
@@ -17,6 +17,7 @@
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Pass/Pass.h"
+#include "llvm/Support/raw_ostream.h"
#define DEBUG_TYPE "test-loop-fusion"
@@ -52,6 +53,16 @@ struct TestLoopFusion
*this, "test-loop-fusion-utilities",
llvm::cl::desc("Enable testing of loop fusion transformation utilities"),
llvm::cl::init(false)};
+
+ Option<bool> clTestEdgeKinds{
+ *this, "test-loop-fusion-edge-kinds",
+ llvm::cl::desc("Enable testing of dependence graph edge kinds"),
+ llvm::cl::init(false)};
+
+ Option<bool> clTestFusionFailure{
+ *this, "test-loop-fusion-failure",
+ llvm::cl::desc("Enable testing of failed producer-consumer fusion"),
+ llvm::cl::init(false)};
};
} // namespace
@@ -154,6 +165,98 @@ static bool testLoopFusionUtilities(AffineForOp forOpA, AffineForOp forOpB,
return false;
}
+// Verify that an unanalyzable dependence is reported by the
+// producer-consumer legality check before slice computation is attempted.
+// This distinguishes a failed dependence proof from a later generic slice
+// failure, which would otherwise leave the same IR unchanged for both cases.
+static bool testFusionFailure(AffineForOp forOpA, AffineForOp forOpB, unsigned,
+ unsigned, unsigned loopDepth,
+ unsigned maxLoopDepth) {
+ if (forOpA->getBlock() != forOpB->getBlock() ||
+ !forOpA->isBeforeInBlock(forOpB))
+ return false;
+
+ FusionStrategy strategy(FusionStrategy::ProducerConsumer);
+ for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
+ ComputationSliceState sliceUnion;
+ FusionResult result =
+ canFuseLoops(forOpA, forOpB, d, &sliceUnion, strategy);
+ if (result.value == FusionResult::FailFusionDependence) {
+ forOpA->emitRemark("fusion dependence prevents fusion at depth ") << d;
+ return false;
+ }
+ }
+ return false;
+}
+
+// Exercises the distinction between storage and exact-SSA graph edges. Keep
+// both edge kinds on the same value so that a query or count that accidentally
+// assumes one kind cannot pass by observing a different value.
+static bool testEdgeKinds(func::FuncOp funcOp) {
+ Operation *srcOp = nullptr;
+ Operation *dstOp = nullptr;
+ Value memref;
+ for (Operation &op : funcOp.getBody().front()) {
+ if (op.getNumResults() == 0 ||
+ !isa<BaseMemRefType>(op.getResult(0).getType()))
+ continue;
+ if (!srcOp) {
+ srcOp = &op;
+ memref = op.getResult(0);
+ } else {
+ dstOp = &op;
+ break;
+ }
+ }
+
+ if (!srcOp || !dstOp) {
+ funcOp.emitError("edge-kind test requires two memref-producing operations");
+ return false;
+ }
+
+ MemRefDependenceGraph graph(funcOp.getBody().front());
+ unsigned srcId = graph.addNode(srcOp);
+ unsigned dstId = graph.addNode(dstOp);
+
+ auto checkState = [&](StringRef label, bool any, bool memory, bool ssa,
+ unsigned edgeCount) {
+ bool anyEdge = graph.hasEdge(srcId, dstId, memref);
+ bool memoryEdge = graph.hasEdge(srcId, dstId, memref,
+ MemRefDependenceGraph::Edge::Kind::Memory);
+ bool ssaEdge = graph.hasEdge(srcId, dstId, memref,
+ MemRefDependenceGraph::Edge::Kind::SSA);
+ bool result = anyEdge == any && memoryEdge == memory && ssaEdge == ssa &&
+ graph.outEdges.lookup(srcId).size() == edgeCount &&
+ graph.inEdges.lookup(dstId).size() == edgeCount &&
+ graph.memrefEdgeCount.lookup(memref) == edgeCount;
+ llvm::outs() << "edge-kinds " << label << " any=" << (anyEdge ? 1 : 0)
+ << " memory=" << (memoryEdge ? 1 : 0)
+ << " ssa=" << (ssaEdge ? 1 : 0)
+ << " out=" << graph.outEdges.lookup(srcId).size()
+ << " in=" << graph.inEdges.lookup(dstId).size()
+ << " count=" << graph.memrefEdgeCount.lookup(memref) << "\n";
+ return result;
+ };
+
+ graph.addEdge(srcId, dstId, memref, MemRefDependenceGraph::Edge::Kind::SSA);
+ if (!checkState("ssa-only", true, false, true, 1))
+ return false;
+
+ graph.addEdge(srcId, dstId, memref,
+ MemRefDependenceGraph::Edge::Kind::Memory);
+ if (!checkState("both", true, true, true, 2))
+ return false;
+
+ graph.removeEdge(srcId, dstId, memref,
+ MemRefDependenceGraph::Edge::Kind::Memory);
+ if (!checkState("ssa-after-memory-removal", true, false, true, 1))
+ return false;
+
+ graph.removeEdge(srcId, dstId, memref,
+ MemRefDependenceGraph::Edge::Kind::SSA);
+ return checkState("empty", false, false, false, 0);
+}
+
using LoopFunc = function_ref<bool(AffineForOp, AffineForOp, unsigned, unsigned,
unsigned, unsigned)>;
@@ -182,6 +285,16 @@ static bool iterateLoops(ArrayRef<SmallVector<AffineForOp, 2>> depthToLoops,
void TestLoopFusion::runOnOperation() {
std::vector<SmallVector<AffineForOp, 2>> depthToLoops;
+ if (clTestEdgeKinds) {
+ if (!testEdgeKinds(getOperation()))
+ signalPassFailure();
+ return;
+ }
+ if (clTestFusionFailure) {
+ gatherLoops(getOperation(), depthToLoops);
+ iterateLoops(depthToLoops, testFusionFailure);
+ return;
+ }
if (clTestLoopFusionUtilities) {
// Run loop fusion until a fixed point is reached.
do {
More information about the Mlir-commits
mailing list