[Mlir-commits] [mlir] [mlir][bufferization] Relax happens-before analysis for unstructured CF (PR #223006)
Colin De Vlieghere
llvmlistbot at llvm.org
Mon Sep 14 11:52:59 PDT 2026
https://github.com/Cubevoid updated https://github.com/llvm/llvm-project/pull/223006
>From 346c98a32ff2599962d59fe6050f8e66a4b14e09 Mon Sep 17 00:00:00 2001
From: Colin De Vlieghere <cdevlieghere at tesla.com>
Date: Thu, 10 Sep 2026 18:11:40 -0700
Subject: [PATCH] [mlir][bufferization] Relax happens-before analysis
On unstructured control-flow, the one-shot analysis is rather
conservative in regards to computing RaW conflicts, requiring strict
dominance which may not always be satisfied.
Relax this requirement to allow more cases to not be marked as
conflicting. Notably, readers/writers within a region that contains a
"diamond" or multiple loops are now handled.
---
.../Bufferization/Transforms/CMakeLists.txt | 1 +
.../Transforms/OneShotAnalysis.cpp | 186 ++++--
.../one-shot-bufferize-analysis.mlir | 575 ++++++++++++++++++
3 files changed, 720 insertions(+), 42 deletions(-)
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
index 2e6bc4d9503b9..24cb7f757d425 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Bufferization/Transforms/CMakeLists.txt
@@ -27,6 +27,7 @@ add_mlir_dialect_library(MLIRBufferizationTransforms
MLIRBufferizationEnumsIncGen
LINK_LIBS PUBLIC
+ MLIRAnalysis
MLIRArithDialect
MLIRBufferizationDialect
MLIRCallInterfaces
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
index e948a02cc415d..cd5ec4a4e3f33 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
@@ -40,8 +40,10 @@
#include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
+#include <memory>
#include <random>
+#include "mlir/Analysis/CFGLoopInfo.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
@@ -54,8 +56,10 @@
#include "mlir/IR/TypeUtilities.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Interfaces/SubsetOpInterface.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/DebugLog.h"
MLIR_DEFINE_EXPLICIT_TYPE_ID(mlir::bufferization::OneShotAnalysisState)
@@ -267,17 +271,86 @@ static bool isInplaceMemoryWrite(OpOperand &opOperand,
return state.isInPlace(opOperand);
}
-/// Return true if `a` happens before `b`, i.e., `a` or one of its ancestors
-/// properly dominates `b` and `b` is not inside `a`.
-static bool happensBefore(Operation *a, Operation *b,
- const DominanceInfo &domInfo) {
+using CFGLoopInfoCache = DenseMap<Region *, std::unique_ptr<CFGLoopInfo>>;
+
+/// Return cached CFG loop info for `region`, or nullptr for single-block
+/// regions (no unstructured loop nest).
+static CFGLoopInfo *getCFGLoopInfo(Region *region, const DominanceInfo &domInfo,
+ CFGLoopInfoCache &cache) {
+ if (region->hasOneBlock())
+ return nullptr;
+ std::unique_ptr<CFGLoopInfo> &entry = cache[region];
+ if (!entry)
+ entry = std::make_unique<CFGLoopInfo>(domInfo.getDomTree(region));
+ return entry.get();
+}
+
+/// Parent loop headers of `a` and `b`. Paths through these are outer backedges
+/// of a loop nest that contains both blocks; they should not count as
+/// "happens after" for sibling inner loops.
+static SmallPtrSet<Block *, 16>
+collectParentLoopHeaders(Block *aBlock, Block *bBlock, CFGLoopInfo *loopInfo) {
+ SmallPtrSet<Block *, 16> barriers;
+ if (!loopInfo)
+ return barriers;
+ CFGLoop *loopA = loopInfo->getLoopFor(aBlock);
+ CFGLoop *loopB = loopInfo->getLoopFor(bBlock);
+ if (loopA == loopB)
+ return barriers;
+ for (CFGLoop *common = loopInfo->getSmallestCommonLoop(loopA, loopB); common;
+ common = common->getParentLoop()) {
+ if (common == loopA || common == loopB)
+ continue;
+ Block *header = common->getHeader();
+ if (header != aBlock && header != bBlock)
+ barriers.insert(header);
+ }
+ return barriers;
+}
+
+/// Return true if there is a CFG path from `from` to `to` in the same region.
+/// If `useCFGLoops`, ignore paths through parent loop headers. `extraBarriers`
+/// are additional blocks a path must not cross.
+static bool
+hasCFGPath(Block *from, Block *to, const DominanceInfo &domInfo,
+ CFGLoopInfoCache &loopInfoCache, bool useCFGLoops,
+ const SmallPtrSetImpl<Block *> *extraBarriers = nullptr) {
+ assert(from->getParent() == to->getParent() && "expected same region");
+ CFGLoopInfo *loopInfo =
+ useCFGLoops ? getCFGLoopInfo(from->getParent(), domInfo, loopInfoCache)
+ : nullptr;
+ SmallPtrSet<Block *, 16> barriers =
+ collectParentLoopHeaders(to, from, loopInfo);
+ if (extraBarriers)
+ barriers.insert(extraBarriers->begin(), extraBarriers->end());
+ return from->isReachable(to, std::move(barriers));
+}
+
+/// Return true if `a` cannot happen after `b`.
+/// True if `a` properly dominates `b` and `b` is not inside `a`, or if they
+/// are in the same region with no CFG path from `b` to `a`.
+/// If `useCFGLoops`, ignore paths through parent loop headers (only sound when
+/// DEF is on every READ-WRITE cycle). `extraBarriers` are extra blocks a path
+/// must not cross (bbArg defs: re-entering the block is a new definition).
+static bool
+cannotHappenAfter(Operation *a, Operation *b, const DominanceInfo &domInfo,
+ CFGLoopInfoCache &loopInfoCache, bool useCFGLoops,
+ const SmallPtrSetImpl<Block *> *extraBarriers = nullptr) {
do {
// TODO: Instead of isProperAncestor + properlyDominates, we should use
// properlyDominatesImpl(a, b, /*enclosingOpOk=*/false)
if (a->isProperAncestor(b))
return false;
+ // Dominance is a stronger condition than reachability. Prefer using it
+ // since it is cached.
if (domInfo.properlyDominates(a, b))
return true;
+ Block *aBlock = a->getBlock();
+ Block *bBlock = b->getBlock();
+ if (aBlock != bBlock && aBlock->getParent() == bBlock->getParent() &&
+ !hasCFGPath(bBlock, aBlock, domInfo, loopInfoCache, useCFGLoops,
+ extraBarriers))
+ return true;
} while ((a = a->getParentOp()));
return false;
}
@@ -296,7 +369,7 @@ static bool happensBefore(Operation *a, Operation *b,
/// %1 = "writing_op"(%0) : tensor<?xf32> -> tensor<?xf32> // WRITE
///
/// This is no longer true inside loops (or repetitive regions). In such cases,
-/// there may not be a meaningful `happensBefore` relationship because ops
+/// there may not be a meaningful `cannotHappenAfter` relationship because ops
/// could be executed multiple times. E.g.:
///
/// Example 2:
@@ -354,8 +427,9 @@ static bool happensBefore(Operation *a, Operation *b,
/// out RaW conflict due to the ordering of ops.
/// 2. Otherwise: There are no loops that interfere with our analysis; for
/// analysis purposes, we can assume that there are no loops/repetitive
-/// regions. I.e., we can rule out a RaW conflict if READ happensBefore WRITE
-/// or WRITE happensBefore DEF. (Checked in `hasReadAfterWriteInterference`.)
+/// regions. I.e., we can rule out a RaW conflict if READ cannot happen after
+/// WRITE or WRITE cannot happen after DEF. (Checked in
+/// `hasReadAfterWriteInterference`.)
///
static bool canUseOpDominanceDueToRegions(OpOperand *uRead, OpOperand *uWrite,
const SetVector<Value> &definitions,
@@ -366,8 +440,8 @@ static bool canUseOpDominanceDueToRegions(OpOperand *uRead, OpOperand *uWrite,
state.getEnclosingRepetitiveRegion(uRead->getOwner(), options);
Region *rDef = state.getEnclosingRepetitiveRegion(def, options);
- // READ and DEF are in the same repetitive region. `happensBefore` can be
- // used to rule out RaW conflicts due to op ordering.
+ // READ and DEF are in the same repetitive region. `cannotHappenAfter` can
+ // be used to rule out RaW conflicts due to op ordering.
if (rRead == rDef)
continue;
@@ -593,11 +667,10 @@ static bool areNonConflictingSubsets(OpOperand *uRead,
/// A conflict is: According to SSA use-def chains, a read R is supposed to read
/// the result of a definition W1. But because of bufferization decisions, R
/// actually reads another definition W2.
-static bool
-hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
- const DenseSet<OpOperand *> &usesWrite,
- const DominanceInfo &domInfo,
- OneShotAnalysisState &state) {
+static bool hasReadAfterWriteInterference(
+ const DenseSet<OpOperand *> &usesRead,
+ const DenseSet<OpOperand *> &usesWrite, const DominanceInfo &domInfo,
+ CFGLoopInfoCache &loopInfoCache, OneShotAnalysisState &state) {
const BufferizationOptions &options = state.getOptions();
// Before going through the main RaW analysis, find cases where a buffer must
@@ -658,6 +731,13 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
continue;
}
+ // Re-entering a bbArg's block is a new definition, not read-after-write
+ // of the previous one.
+ SmallPtrSet<Block *, 8> bbArgDefBlocks;
+ for (Value def : definitions)
+ if (isa<BlockArgument>(def))
+ bbArgDefBlocks.insert(def.getParentBlock());
+
// Look for conflicting memory writes. Potential conflicts are writes to an
// alias that have been decided to bufferize inplace.
for (OpOperand *uConflictingWrite : usesWrite) {
@@ -679,14 +759,16 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
// Inside of repetitive regions, ops may be executed multiple times and op
// dominance cannot be used to rule out conflicts.
if (useDominance) {
- // No conflict if the readingOp dominates conflictingWritingOp, i.e.,
- // the write is not visible when reading.
+ // No conflict if the readingOp cannot happen after
+ // conflictingWritingOp, i.e., the write is not visible when reading.
//
// Note: If ops are executed multiple times (e.g., because they are
- // inside a loop), there may be no meaningful `happensBefore`
+ // inside a loop), there may be no meaningful `cannotHappenAfter`
// relationship.
- if (happensBefore(readingOp, conflictingWritingOp, domInfo)) {
- LDBG() << " no conflict: read happens before write";
+ if (cannotHappenAfter(readingOp, conflictingWritingOp, domInfo,
+ loopInfoCache,
+ /*useCFGLoops=*/true, &bbArgDefBlocks)) {
+ LDBG() << " no conflict: read cannot happen after write";
continue;
}
@@ -762,11 +844,12 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
for (Value definition : definitions) {
LDBG() << " * definition = " << definition;
- // No conflict if the conflicting write happens before the definition.
+ // No conflict if the conflicting write cannot happen after the
+ // definition.
if (Operation *defOp = definition.getDefiningOp()) {
- if (happensBefore(conflictingWritingOp, defOp, domInfo)) {
- // conflictingWritingOp happens before defOp. No conflict.
- LDBG() << " no conflict: write happens before definition";
+ if (cannotHappenAfter(conflictingWritingOp, defOp, domInfo,
+ loopInfoCache, useDominance)) {
+ LDBG() << " no conflict: write cannot happen after definition";
continue;
}
// No conflict if conflictingWritingOp is contained in defOp.
@@ -775,13 +858,18 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
continue;
}
} else {
- auto bbArg = cast<BlockArgument>(definition);
- Block *block = bbArg.getOwner();
- if (!block->findAncestorOpInBlock(*conflictingWritingOp)) {
- LDBG() << " no conflict: definition is bbArg "
- "and write happens outside of block";
- // conflictingWritingOp happens outside of the block. No
- // conflict.
+ // A bbArg is defined at block entry. The write happens after that
+ // only if it is in the same block or in a reachable block (e.g. a
+ // successor).
+ Block *defBlock = cast<BlockArgument>(definition).getOwner();
+ Operation *writeOp = conflictingWritingOp;
+ while (writeOp && writeOp->getParentRegion() != defBlock->getParent())
+ writeOp = writeOp->getParentOp();
+ if (!writeOp || (writeOp->getBlock() != defBlock &&
+ !hasCFGPath(defBlock, writeOp->getBlock(), domInfo,
+ loopInfoCache, useDominance))) {
+ LDBG() << " no conflict: definition is bbarg and write cannot "
+ "happen after def";
continue;
}
}
@@ -887,7 +975,8 @@ static void getAliasingReads(DenseSet<OpOperand *> &res, Value root,
/// involving aliases of the given OpOperand are checked.
static bool wouldCreateReadAfterWriteInterference(
OpOperand &operand, const DominanceInfo &domInfo,
- OneShotAnalysisState &state, bool checkConsistencyOnly = false) {
+ CFGLoopInfoCache &loopInfoCache, OneShotAnalysisState &state,
+ bool checkConsistencyOnly = false) {
// Collect reads and writes of all aliases of OpOperand and OpResult.
DenseSet<OpOperand *> usesRead, usesWrite;
getAliasingReads(usesRead, operand.get(), state);
@@ -899,7 +988,8 @@ static bool wouldCreateReadAfterWriteInterference(
if (!checkConsistencyOnly && state.bufferizesToMemoryWrite(operand))
usesWrite.insert(&operand);
- return hasReadAfterWriteInterference(usesRead, usesWrite, domInfo, state);
+ return hasReadAfterWriteInterference(usesRead, usesWrite, domInfo,
+ loopInfoCache, state);
}
/// Annotate IR with details about the detected non-writability conflict.
@@ -989,16 +1079,17 @@ void OneShotAnalysisState::resetCache() {
}
/// Determine if `operand` can be bufferized in-place.
-static LogicalResult
-bufferizableInPlaceAnalysisImpl(OpOperand &operand, OneShotAnalysisState &state,
- const DominanceInfo &domInfo) {
+static LogicalResult bufferizableInPlaceAnalysisImpl(
+ OpOperand &operand, const DominanceInfo &domInfo,
+ CFGLoopInfoCache &loopInfoCache, OneShotAnalysisState &state) {
LDBG() << "//===-------------------------------------------===//\n"
<< "Analyzing operand #" << operand.getOperandNumber() << " of "
<< OpWithFlags(operand.getOwner(), OpPrintingFlags().skipRegions());
bool foundInterference =
wouldCreateWriteToNonWritableBuffer(operand, state) ||
- wouldCreateReadAfterWriteInterference(operand, domInfo, state);
+ wouldCreateReadAfterWriteInterference(operand, domInfo, loopInfoCache,
+ state);
if (foundInterference)
state.bufferizeOutOfPlace(operand);
@@ -1009,16 +1100,25 @@ bufferizableInPlaceAnalysisImpl(OpOperand &operand, OneShotAnalysisState &state,
return success();
}
-LogicalResult
-OneShotAnalysisState::analyzeSingleOp(Operation *op,
- const DominanceInfo &domInfo) {
+static LogicalResult analyzeSingleOpImpl(Operation *op,
+ const DominanceInfo &domInfo,
+ CFGLoopInfoCache &loopInfoCache,
+ OneShotAnalysisState &state) {
for (OpOperand &opOperand : op->getOpOperands())
if (isa<TensorLikeType>(opOperand.get().getType()))
- if (failed(bufferizableInPlaceAnalysisImpl(opOperand, *this, domInfo)))
+ if (failed(bufferizableInPlaceAnalysisImpl(opOperand, domInfo,
+ loopInfoCache, state)))
return failure();
return success();
}
+LogicalResult
+OneShotAnalysisState::analyzeSingleOp(Operation *op,
+ const DominanceInfo &domInfo) {
+ CFGLoopInfoCache loopInfoCache;
+ return analyzeSingleOpImpl(op, domInfo, loopInfoCache, *this);
+}
+
/// Analyze equivalence of tied OpResult/OpOperand pairs of the given ops.
static void equivalenceAnalysis(SmallVector<Operation *> &ops,
OneShotAnalysisState &state) {
@@ -1177,8 +1277,9 @@ LogicalResult OneShotAnalysisState::analyzeOp(Operation *op,
}
// Analyze ops in the computed order.
+ CFGLoopInfoCache loopInfoCache;
for (Operation *op : orderedOps)
- if (failed(analyzeSingleOp(op, domInfo)))
+ if (failed(analyzeSingleOpImpl(op, domInfo, loopInfoCache, *this)))
return failure();
equivalenceAnalysis(op, *this);
@@ -1214,6 +1315,7 @@ LogicalResult bufferization::checkPreBufferizationAssumptions(
if (walkResult.wasInterrupted())
return failure();
+ CFGLoopInfoCache loopInfoCache;
walkResult = op->walk([&](BufferizableOpInterface op) {
// Skip ops that are not in the filter.
if (!options.isOpAllowed(op.getOperation()))
@@ -1233,7 +1335,7 @@ LogicalResult bufferization::checkPreBufferizationAssumptions(
for (OpOperand &opOperand : op->getOpOperands()) {
if (isa<TensorLikeType>(opOperand.get().getType())) {
if (wouldCreateReadAfterWriteInterference(
- opOperand, domInfo, state,
+ opOperand, domInfo, loopInfoCache, state,
/*checkConsistencyOnly=*/true)) {
// This error can happen if certain "mustBufferizeInPlace" interface
// methods are implemented incorrectly, such that the IR already has
diff --git a/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
index bc8ac8d559aa4..e476e8ae2f090 100644
--- a/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
+++ b/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
@@ -219,3 +219,578 @@ func.func @looping_branches_sequence_inside_def(%f: f32) {
^bb3:
func.return
}
+
+// -----
+
+// Sequential loops; loop 2 forwards loop 1's bbarg. Inserts are inplace.
+// CHECK-LABEL: func @sequential_loops_forward_bbarg(
+func.func @sequential_loops_forward_bbarg() -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+ %enter = "test.foo"() : () -> (i1)
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+ cf.cond_br %enter, ^bb1(%init : tensor<5xf32>), ^bb1(%init : tensor<5xf32>)
+
+^bb1(%cache: tensor<5xf32>):
+ %c1 = "test.foo"() : () -> (i1)
+ cf.cond_br %c1, ^bb3, ^bb2
+
+^bb2:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %mid = tensor.insert %val into %cache[%pos] : tensor<5xf32>
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %updated1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+ cf.cond_br %enter, ^bb1(%updated1 : tensor<5xf32>), ^bb1(%updated1 : tensor<5xf32>)
+
+^bb3:
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+ cf.cond_br %enter, ^bb4(%cache : tensor<5xf32>), ^bb4(%cache : tensor<5xf32>)
+
+^bb4(%cache2: tensor<5xf32>):
+ %c2 = "test.foo"() : () -> (i1)
+ cf.cond_br %c2, ^bb6, ^bb5
+
+^bb5:
+ %pos3 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %updated2 = tensor.insert %val3 into %cache2[%pos3] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+ cf.cond_br %enter, ^bb4(%updated2 : tensor<5xf32>), ^bb4(%updated2 : tensor<5xf32>)
+
+^bb6:
+ func.return %cache2 : tensor<5xf32>
+}
+
+// -----
+
+// Diamond: read vs write of an OpResult dest. Insert is inplace.
+// CHECK-LABEL: func @diamond_read_write(
+func.func @diamond_read_write(%c: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+ %pos = "test.foo"() : () -> (index)
+// CHECK: tensor.extract
+// CHECK-SAME: __inplace_operands_attr__ = ["true", "none"]
+ %extracted = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%extracted) : (f32) -> ()
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+ cf.br ^bb6(%t : tensor<5xf32>)
+^bb5:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+ cf.br ^bb6(%inserted : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Reads in both arms, write after the join. Insert is inplace.
+// CHECK-LABEL: func @read_then_write_after_join(
+func.func @read_then_write_after_join(%c: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+ %pos = "test.foo"() : () -> (index)
+ %e1 = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e1) : (f32) -> ()
+ cf.br ^bb3
+^bb2:
+ %pos2 = "test.foo"() : () -> (index)
+ %e2 = tensor.extract %t[%pos2] : tensor<5xf32>
+ "test.qux"(%e2) : (f32) -> ()
+ cf.br ^bb3
+^bb3:
+ %pos3 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val into %t[%pos3] : tensor<5xf32>
+ func.return %inserted : tensor<5xf32>
+}
+
+// -----
+
+// Write in one arm, extract of the original after the join. Insert is out-of-place.
+// CHECK-LABEL: func @write_then_read_after_join(
+func.func @write_then_read_after_join(%c: i1, %f: f32) -> f32 {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %inserted = tensor.insert %val into %t[%pos] : tensor<5xf32>
+ cf.br ^bb3(%inserted : tensor<5xf32>)
+^bb2:
+ cf.br ^bb3(%t : tensor<5xf32>)
+^bb3(%r: tensor<5xf32>):
+ %pos2 = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos2] : tensor<5xf32>
+ func.return %e : f32
+}
+
+// -----
+
+// bb0 -> bb2|bb5, bb2 -> bb5|bb6. Read in bb2, write in bb5. Insert is inplace.
+// CHECK-LABEL: func @read_predecessor_write_successor(
+func.func @read_predecessor_write_successor(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+ %pos = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.cond_br %c2, ^bb5, ^bb6(%t : tensor<5xf32>)
+^bb5:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+ cf.br ^bb6(%inserted : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Same CFG; write in bb2, extract of the original in bb5. Insert is out-of-place.
+// CHECK-LABEL: func @write_predecessor_read_successor(
+func.func @write_predecessor_read_successor(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %inserted = tensor.insert %val into %t[%pos] : tensor<5xf32>
+ cf.cond_br %c2, ^bb5, ^bb6(%inserted : tensor<5xf32>)
+^bb5:
+ %pos2 = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos2] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^bb6(%t : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Three exclusive arms: read, write, passthrough. Insert is inplace.
+// CHECK-LABEL: func @three_way_exclusive_read_write(
+func.func @three_way_exclusive_read_write(%c1: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c1, ^read, ^rest
+^rest:
+ cf.cond_br %c2, ^write, ^pass
+^read:
+ %pos = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^join(%t : tensor<5xf32>)
+^write:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+ cf.br ^join(%inserted : tensor<5xf32>)
+^pass:
+ cf.br ^join(%t : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Both diamond arms write the same dest. Both inserts are inplace.
+// CHECK-LABEL: func @writes_in_both_diamond_arms(
+func.func @writes_in_both_diamond_arms(%c: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %i1 = tensor.insert %val into %t[%pos] : tensor<5xf32>
+ cf.br ^bb3(%i1 : tensor<5xf32>)
+^bb2:
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %i2 = tensor.insert %val2 into %t[%pos2] : tensor<5xf32>
+ cf.br ^bb3(%i2 : tensor<5xf32>)
+^bb3(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Nested diamond: inner read vs outer write. Insert is inplace.
+// CHECK-LABEL: func @nested_diamond_exclusive(
+func.func @nested_diamond_exclusive(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^outer_read, ^outer_write
+^outer_read:
+ cf.cond_br %c2, ^inner_read, ^inner_pass
+^inner_read:
+ %pos = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^join(%t : tensor<5xf32>)
+^inner_pass:
+ cf.br ^join(%t : tensor<5xf32>)
+^outer_write:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+ cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Diamond inside a loop; dest defined outside. Insert is out-of-place.
+// CHECK-LABEL: func @diamond_inside_loop_outside_def(
+func.func @diamond_inside_loop_outside_def(%c: i1, %back: i1, %f: f32) -> tensor<5xf32> {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.br ^header
+^header:
+ cf.cond_br %c, ^read, ^write
+^read:
+ %pos = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^join
+^write:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+ cf.br ^join
+^join:
+ cf.cond_br %back, ^header, ^exit
+^exit:
+ func.return %t : tensor<5xf32>
+}
+
+// -----
+
+// Diamond inside a loop; dest defined in the loop body. Insert is out-of-place.
+// CHECK-LABEL: func @diamond_inside_loop_inside_def(
+func.func @diamond_inside_loop_inside_def(%c: i1, %back: i1, %f: f32) -> tensor<5xf32> {
+ cf.br ^header
+^header:
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.cond_br %c, ^read, ^write
+^read:
+ %pos = "test.foo"() : () -> (index)
+ %e = tensor.extract %t[%pos] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^join(%t : tensor<5xf32>)
+^write:
+ %pos2 = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+ cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+ cf.cond_br %back, ^header, ^exit(%r : tensor<5xf32>)
+^exit(%out: tensor<5xf32>):
+ func.return %out : tensor<5xf32>
+}
+
+// -----
+
+// Loop then diamond on the forwarded bbarg. Inserts are inplace.
+// CHECK-LABEL: func @sequential_loop_then_diamond(
+func.func @sequential_loop_then_diamond(%c: i1, %enter: i1) -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+ cf.cond_br %enter, ^loop(%init : tensor<5xf32>), ^loop(%init : tensor<5xf32>)
+^loop(%cache: tensor<5xf32>):
+ %c1 = "test.foo"() : () -> (i1)
+ cf.cond_br %c1, ^after, ^body
+^body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %mid = tensor.insert %val into %cache[%pos] : tensor<5xf32>
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %updated = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+ cf.cond_br %enter, ^loop(%updated : tensor<5xf32>), ^loop(%updated : tensor<5xf32>)
+^after:
+ cf.cond_br %c, ^read(%cache : tensor<5xf32>), ^write(%cache : tensor<5xf32>)
+^read(%r0: tensor<5xf32>):
+ %pos3 = "test.foo"() : () -> (index)
+ %e = tensor.extract %r0[%pos3] : tensor<5xf32>
+ "test.qux"(%e) : (f32) -> ()
+ cf.br ^join(%r0 : tensor<5xf32>)
+^write(%r1: tensor<5xf32>):
+ %pos4 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %inserted = tensor.insert %val3 into %r1[%pos4] : tensor<5xf32>
+ cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+ func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Sequential inner loops inside an outer loop. Inserts are inplace.
+// CHECK-LABEL: func @outer_loop_sequential_inner_loops(
+func.func @outer_loop_sequential_inner_loops(%again: i1) -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+ cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+ cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+ %e1 = "test.foo"() : () -> (i1)
+ cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+ cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+ cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+ %e2 = "test.foo"() : () -> (i1)
+ cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+ %pos3 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+ cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+ cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+ func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// Sequential inner loops inside an outer loop; dest defined outside. The
+// outer backedge lets l1 write and l2 read the same outside value on
+// different iterations, so the insert dest is out-of-place.
+// CHECK-LABEL: func @outer_loop_seq_inners_outside_def(
+func.func @outer_loop_seq_inners_outside_def(%c: i1, %back: i1, %f: f32) -> f32 {
+ %empty = tensor.empty() : tensor<5xf32>
+ %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+ cf.br ^outer
+^outer:
+ cf.br ^l1
+^l1:
+ %e1 = "test.foo"() : () -> (i1)
+ cf.cond_br %e1, ^l2, ^l1body
+^l1body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %mid = tensor.insert %val into %t[%pos] : tensor<5xf32>
+ cf.br ^l1
+^l2:
+ %e2 = "test.foo"() : () -> (i1)
+ cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+ %pos2 = "test.foo"() : () -> (index)
+ %ex = tensor.extract %t[%pos2] : tensor<5xf32>
+ "test.qux"(%ex) : (f32) -> ()
+ cf.br ^l2
+^outer_end:
+ cf.cond_br %back, ^outer, ^exit
+^exit:
+ %pos3 = "test.foo"() : () -> (index)
+ %r = tensor.extract %t[%pos3] : tensor<5xf32>
+ func.return %r : f32
+}
+
+// -----
+
+// l1 writes the header bbarg then extracts it. The extract must observe %c1
+// before the insert, so the insert dest is out-of-place.
+// CHECK-LABEL: func @l1_write_then_extract_bbarg(
+func.func @l1_write_then_extract_bbarg(%again: i1) -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+ cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+ cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+ %e1 = "test.foo"() : () -> (i1)
+ cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %upd1 = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+ %posr = "test.foo"() : () -> (index)
+ %old = tensor.extract %c1[%posr] : tensor<5xf32>
+ "test.qux"(%old) : (f32) -> ()
+ cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+ cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+ %e2 = "test.foo"() : () -> (i1)
+ cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+ %pos3 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+ cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+ cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+ func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// Extra edge l2body -> l1body, not through the outer header. The tensor is
+// copied on the branches into the inner loops.
+// CHECK-LABEL: func @irreducible_l2_to_l1(
+func.func @irreducible_l2_to_l1(%again: i1, %weird: i1) -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+ cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["false"]}
+ cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+ %e1 = "test.foo"() : () -> (i1)
+ cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+ cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+ cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+ %e2 = "test.foo"() : () -> (i1)
+ cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+ %pos3 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+ cf.cond_br %weird, ^l1body, ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+ cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+ func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// l1 writes the carried dest then extracts the outer bbarg. The extract must
+// observe %cache before those writes, so the insert dests are out-of-place.
+// CHECK-LABEL: func @l1_write_then_extract_carried(
+func.func @l1_write_then_extract_carried(%again: i1) -> tensor<5xf32> {
+ %init = tensor.empty() : tensor<5xf32>
+ cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+ cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+ %e1 = "test.foo"() : () -> (i1)
+ cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+ %pos = "test.foo"() : () -> (index)
+ %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+ %pos2 = "test.foo"() : () -> (index)
+ %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+ %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+ %posr = "test.foo"() : () -> (index)
+ %old = tensor.extract %cache[%posr] : tensor<5xf32>
+ "test.qux"(%old) : (f32) -> ()
+ cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+ cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+ %e2 = "test.foo"() : () -> (i1)
+ cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+ %pos3 = "test.foo"() : () -> (index)
+ %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+ %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+ cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+ cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+ func.return %c2 : tensor<5xf32>
+}
More information about the Mlir-commits
mailing list