[Mlir-commits] [mlir] 606e2ac - [mlir][bufferization] Relax happens-before analysis (#223777)

[llvmlistbot at llvm.org]

Author: Colin De Vlieghere
Date: 2026-09-24T11:05:13-07:00
New Revision: 606e2acdd1cd869831600b001d76dbe123e80e2c

URL: https://github.com/llvm/llvm-project/commit/606e2acdd1cd869831600b001d76dbe123e80e2c
DIFF: https://github.com/llvm/llvm-project/commit/606e2acdd1cd869831600b001d76dbe123e80e2c.diff

LOG: [mlir][bufferization] Relax happens-before analysis (#223777)

On unstructured control-flow, the one-shot analysis is rather
conservative in regards to computing RaW conflicts, requiring strict
dominance which may not always be satisfied.

Relax this requirement to allow more cases to not be marked as
conflicting. Notably, readers/writers within a region that contains a
"diamond" or multiple loops are now handled.

This is a simpler version of
https://github.com/llvm/llvm-project/pull/223006 without the CFGLoopInfo
analysis, meaning it is a bit more conservative when analyzing regions
with nested loops.

Added: 
    

Modified: 
    mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h
    mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
    mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir

Removed: 
    


################################################################################
diff  --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h
index 49706c5614d83..af5d2bd9a2db4 100644
--- a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h
+++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h
@@ -11,6 +11,9 @@
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include <memory>
+#include <optional>
 #include <string>
 
 namespace mlir {
@@ -52,6 +55,12 @@ struct OneShotBufferizationOptions : public BufferizationOptions {
   /// `AnalysisHeuristic::Fuzzer`. The fuzzer should be used only with
   /// `testAnalysisOnly = true`.
   unsigned analysisFuzzerSeed = 0;
+
+  /// Whether the IR contains a region with more than one block. When unset,
+  /// the analysis walks the IR to compute it.
+  /// Note: If the IR contains unstructured control flow, but this flag is set
+  /// to "false", the bufferization may produce incorrect IR.
+  std::optional<bool> mayHaveUnstructuredControlFlow = std::nullopt;
 };
 
 /// State for analysis-enabled bufferization. This class keeps track of alias
@@ -66,7 +75,7 @@ class OneShotAnalysisState : public AnalysisState {
 
   OneShotAnalysisState(const OneShotAnalysisState &) = delete;
 
-  ~OneShotAnalysisState() override = default;
+  ~OneShotAnalysisState() override;
 
   static bool classof(const AnalysisState *base) {
     return base->getType() == TypeID::get<OneShotAnalysisState>();
@@ -78,6 +87,10 @@ class OneShotAnalysisState : public AnalysisState {
         AnalysisState::getOptions());
   }
 
+  /// True if any region in the analyzed IR has more than one block. Taken from
+  /// the options when set; otherwise computed by walking the IR.
+  bool mayHaveUnstructuredControlFlow() const { return mayHaveUnstructuredCF; }
+
   /// Analyze the given op and its nested ops.
   LogicalResult analyzeOp(Operation *op, const DominanceInfo &domInfo);
 
@@ -131,6 +144,20 @@ class OneShotAnalysisState : public AnalysisState {
   /// retrieve them from the cache.
   const SetVector<Value> &findDefinitionsCached(OpOperand *opOperand);
 
+  /// Return true if `to` is reachable from `from` without crossing `barriers`.
+  /// Results are cached; the cache is cleared by `resetCache`.
+  bool
+  isReachableCached(Block *from, Block *to,
+                    const llvm::SmallPtrSetImpl<Block *> *barriers = nullptr);
+
+  /// Cached `canUseOpDominanceDueToBlocks`. The result depends only on the
+  /// blocks that contain the read, the write, and the definitions, not on the
+  /// ops themselves. `defBlocks` must be sorted and unique. `compute` runs
+  /// only on a cache miss. Cleared by `resetCache`.
+  bool canUseOpDominanceDueToBlocksCached(Block *readBlock, Block *writeBlock,
+                                          llvm::ArrayRef<Block *> defBlocks,
+                                          function_ref<bool()> compute);
+
   /// Return whether `uRead` and `uConflictingWrite` are non-conflicting
   /// subsets, with caching.
   bool areNonConflictingSubsetsCached(OpOperand *uRead,
@@ -237,6 +264,18 @@ class OneShotAnalysisState : public AnalysisState {
   /// Cache definitions of tensor values.
   DenseMap<Value, SetVector<Value>> cachedDefinitions;
 
+  /// True if any region has more than one block.
+  bool mayHaveUnstructuredCF = false;
+
+  /// Cached CFG reachability. Defined out-of-line to keep BitVector out of
+  /// this header.
+  class CFGReachabilityCache;
+  std::unique_ptr<CFGReachabilityCache> cfgReachabilityCache;
+
+  /// Cached block-granularity op-dominance decisions. Defined out-of-line.
+  class OpDominanceBlockCache;
+  std::unique_ptr<OpDominanceBlockCache> opDominanceBlockCache;
+
   /// Cache results of areNonConflictingSubsets checks. The bool value is `true`
   /// if the operands are non-conflicting subsets, `false` if they are
   /// conflicting. The absence of an entry means uncached.

diff  --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
index e948a02cc415d..19e8d61ffffd0 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
@@ -41,6 +41,7 @@
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 
 #include <random>
+#include <tuple>
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
@@ -54,8 +55,12 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/DebugLog.h"
 
 MLIR_DEFINE_EXPLICIT_TYPE_ID(mlir::bufferization::OneShotAnalysisState)
@@ -112,9 +117,26 @@ static void setInPlaceOpOperand(OpOperand &opOperand, bool inPlace) {
 // OneShotAnalysisState
 //===----------------------------------------------------------------------===//
 
+/// A region with more than one block is unstructured control flow. Single-block
+/// regions, including structured loops, are not.
+static bool detectUnstructuredControlFlow(Operation *op) {
+  WalkResult walkRes = op->walk([&](Operation *nested) {
+    for (Region &region : nested->getRegions()) {
+      if (region.getBlocks().size() > 1) {
+        return WalkResult::interrupt();
+      }
+    }
+    return WalkResult::advance();
+  });
+  return walkRes.wasInterrupted();
+}
+
 OneShotAnalysisState::OneShotAnalysisState(
     Operation *op, const OneShotBufferizationOptions &options)
     : AnalysisState(options, TypeID::get<OneShotAnalysisState>()) {
+  mayHaveUnstructuredCF = options.mayHaveUnstructuredControlFlow.value_or(
+      detectUnstructuredControlFlow(op));
+
   // Set up alias sets.
   op->walk([&](Operation *op) {
     for (Value v : op->getResults())
@@ -267,17 +289,140 @@ static bool isInplaceMemoryWrite(OpOperand &opOperand,
   return state.isInPlace(opOperand);
 }
 
-/// Return true if `a` happens before `b`, i.e., `a` or one of its ancestors
-/// properly dominates `b` and `b` is not inside `a`.
-static bool happensBefore(Operation *a, Operation *b,
-                          const DominanceInfo &domInfo) {
+/// Cached CFG reachability to avoid repeated linear BFS traversals from
+/// `Block->isReachable`.
+class OneShotAnalysisState::CFGReachabilityCache {
+public:
+  bool isReachable(Block *from, Block *to,
+                   const SmallPtrSetImpl<Block *> *barriers = nullptr) {
+    assert(from->getParent() == to->getParent() && "expected same region");
+
+    SmallPtrSet<Block *, 16> except;
+    BarrierKey sortedBarriers;
+    if (barriers && !barriers->empty()) {
+      except.insert(barriers->begin(), barriers->end());
+      sortedBarriers.append(barriers->begin(), barriers->end());
+      llvm::sort(sortedBarriers);
+    }
+
+    ReachableByBarriers &inner = cached[from];
+    if (auto it = inner.find(sortedBarriers); it != inner.end())
+      return it->second.contains(to);
+
+    // Same traversal as `Block::isReachable`: `except` is both the barrier set
+    // and the visited set.
+    ReachableSet reachable;
+    SmallVector<Block *> worklist(from->succ_begin(), from->succ_end());
+    while (!worklist.empty()) {
+      Block *next = worklist.pop_back_val();
+      if (!except.insert(next).second)
+        continue;
+      reachable.insert(next);
+      worklist.append(next->succ_begin(), next->succ_end());
+    }
+    bool result = reachable.contains(to);
+    inner.try_emplace(std::move(sortedBarriers), std::move(reachable));
+    return result;
+  }
+
+  void clear() { cached.clear(); }
+
+private:
+  using ReachableSet = SmallPtrSet<Block *, 16>;
+  /// Sorted extra-barrier blocks for one query. Empty means no extra barriers.
+  using BarrierKey = SmallVector<Block *, 8>;
+  /// Sorted barrier list -> blocks reachable from `from` without crossing those
+  /// barriers.
+  using ReachableByBarriers = DenseMap<BarrierKey, ReachableSet>;
+  /// `from` block -> reachable-set keyed by sorted extra-barrier blocks.
+  DenseMap<Block *, ReachableByBarriers> cached;
+};
+
+/// `canUseOpDominanceDueToBlocks` walks enclosing regions and queries CFG
+/// reachability, but the answer is a property of blocks: every op in a block
+/// has the same parent chain, and a definition contributes only its block as
+/// a reachability barrier. One entry per (read block, write block, def blocks)
+/// therefore covers every operand pair in those blocks.
+class OneShotAnalysisState::OpDominanceBlockCache {
+public:
+  bool getOrCompute(Block *readBlock, Block *writeBlock,
+                    ArrayRef<Block *> sortedDefBlocks,
+                    function_ref<bool()> compute) {
+    if (sortedDefBlocks.size() == 1) {
+      auto key =
+          std::make_tuple(readBlock, writeBlock, sortedDefBlocks.front());
+      if (auto it = singleDef.find(key); it != singleDef.end())
+        return it->second;
+      bool result = compute();
+      singleDef.try_emplace(key, result);
+      return result;
+    }
+
+    DefKey key(sortedDefBlocks.begin(), sortedDefBlocks.end());
+    auto &byDefs = manyDefs[std::make_pair(readBlock, writeBlock)];
+    if (auto it = byDefs.find(key); it != byDefs.end())
+      return it->second;
+    bool result = compute();
+    byDefs.try_emplace(std::move(key), result);
+    return result;
+  }
+
+  void clear() {
+    singleDef.clear();
+    manyDefs.clear();
+  }
+
+private:
+  using DefKey = SmallVector<Block *, 4>;
+  /// Common case: a single definition block.
+  DenseMap<std::tuple<Block *, Block *, Block *>, bool> singleDef;
+  /// Sorted unique definition blocks, for reads with several definitions.
+  DenseMap<std::pair<Block *, Block *>, DenseMap<DefKey, bool>> manyDefs;
+};
+
+OneShotAnalysisState::~OneShotAnalysisState() = default;
+
+/// Return true if `a` cannot happen after `b`.
+/// True if `a` properly dominates `b` and `b` is not inside `a`, or if they
+/// are in the same region with no CFG path from `b` to `a`.
+/// Dominance is sufficient but not necessary. E.g.:
+///
+/// ^bb0:
+///   cf.cond_br %c, ^bb1, ^bb2
+/// ^bb1:
+///   "op_a"(%t)
+///   cf.br ^bb3
+/// ^bb2:
+///   "op_b"(%t)
+///   cf.br ^bb3
+/// ^bb3:
+///
+/// `op_a` does not dominate `op_b`, but there is no CFG path from `op_b` to
+/// `op_a`, so `op_a` cannot happen after `op_b`.
+///
+/// `extraBarriers` are extra blocks a CFG path from `b` to `a` must not cross.
+static bool cannotHappenAfter(Operation *a, Operation *b,
+                              const DominanceInfo &domInfo,
+                              OneShotAnalysisState &state,
+                              SmallPtrSet<Block *, 16> extraBarriers = {}) {
   do {
     // TODO: Instead of isProperAncestor + properlyDominates, we should use
     // properlyDominatesImpl(a, b, /*enclosingOpOk=*/false)
     if (a->isProperAncestor(b))
       return false;
+    // Dominance is a stronger condition than reachability. Prefer using it
+    // since it is cached and works within a single block.
     if (domInfo.properlyDominates(a, b))
       return true;
+    // Distinct blocks in the same region only exist with unstructured control
+    // flow. Dominance is a complete ordering otherwise.
+    if (state.mayHaveUnstructuredControlFlow()) {
+      Block *aBlock = a->getBlock();
+      Block *bBlock = b->getBlock();
+      if (aBlock != bBlock && aBlock->getParent() == bBlock->getParent() &&
+          !state.isReachableCached(bBlock, aBlock, &extraBarriers))
+        return true;
+    }
   } while ((a = a->getParentOp()));
   return false;
 }
@@ -296,7 +441,7 @@ static bool happensBefore(Operation *a, Operation *b,
 /// %1 = "writing_op"(%0) : tensor<?xf32> -> tensor<?xf32>  // WRITE
 ///
 /// This is no longer true inside loops (or repetitive regions). In such cases,
-/// there may not be a meaningful `happensBefore` relationship because ops
+/// there may not be a meaningful `cannotHappenAfter` relationship because ops
 /// could be executed multiple times. E.g.:
 ///
 /// Example 2:
@@ -354,8 +499,9 @@ static bool happensBefore(Operation *a, Operation *b,
 ///    out RaW conflict due to the ordering of ops.
 /// 2. Otherwise: There are no loops that interfere with our analysis; for
 ///    analysis purposes, we can assume that there are no loops/repetitive
-///    regions. I.e., we can rule out a RaW conflict if READ happensBefore WRITE
-///    or WRITE happensBefore DEF. (Checked in `hasReadAfterWriteInterference`.)
+///    regions. I.e., we can rule out a RaW conflict if READ cannot happen after
+///    WRITE or WRITE cannot happen after DEF. (Checked in
+///    `hasReadAfterWriteInterference`.)
 ///
 static bool canUseOpDominanceDueToRegions(OpOperand *uRead, OpOperand *uWrite,
                                           const SetVector<Value> &definitions,
@@ -366,8 +512,8 @@ static bool canUseOpDominanceDueToRegions(OpOperand *uRead, OpOperand *uWrite,
         state.getEnclosingRepetitiveRegion(uRead->getOwner(), options);
     Region *rDef = state.getEnclosingRepetitiveRegion(def, options);
 
-    // READ and DEF are in the same repetitive region. `happensBefore` can be
-    // used to rule out RaW conflicts due to op ordering.
+    // READ and DEF are in the same repetitive region. `cannotHappenAfter` can
+    // be used to rule out RaW conflicts due to op ordering.
     if (rRead == rDef)
       continue;
 
@@ -402,32 +548,76 @@ static bool canUseOpDominanceDueToRegions(OpOperand *uRead, OpOperand *uWrite,
 ///
 /// Op dominance cannot be used if there is a path from block(READ) to
 /// block(WRITE) and a path from block(WRITE) to block(READ). block(DEF) should
-/// not appear on that path.
+/// not appear on that path. Walk from the uses up through enclosing regions
+/// and stop at the definition region(s): SSA dominance means a cycle that
+/// skips DEF cannot live above DEF.
+static bool
+computeCanUseOpDominanceDueToBlocks(OpOperand *uRead, OpOperand *uWrite,
+                                    const SetVector<Value> &definitions,
+                                    OneShotAnalysisState &state) {
+  // Outermost region that contains a definition. SSA dominance means a cycle
+  // that skips DEF cannot live above this.
+  assert(!definitions.empty() && "expected at least one definition");
+  Region *outermostDefRegion = nullptr;
+  for (Value def : definitions) {
+    Region *defRegion = def.getParentRegion();
+    if (!outermostDefRegion || defRegion->isAncestor(outermostDefRegion))
+      outermostDefRegion = defRegion;
+  }
+  assert(outermostDefRegion && "expected a definition region");
+
+  for (Operation *readOp = uRead->getOwner(); readOp != nullptr;
+       readOp = readOp->getParentOp()) {
+    Region *region = readOp->getParentRegion();
+    if (!region)
+      continue;
+
+    Operation *writeOp = region->findAncestorOpInRegion(*uWrite->getOwner());
+    if (!writeOp)
+      continue;
+
+    Block *readBlock = readOp->getBlock();
+    Block *writeBlock = writeOp->getBlock();
+    for (Value def : definitions) {
+      SmallPtrSet<Block *, 16> except = {def.getParentBlock()};
+      if (state.isReachableCached(readBlock, writeBlock, &except) &&
+          state.isReachableCached(writeBlock, readBlock, &except))
+        return false;
+    }
+    if (region == outermostDefRegion)
+      break;
+  }
+  return true;
+}
+
 static bool canUseOpDominanceDueToBlocks(OpOperand *uRead, OpOperand *uWrite,
                                          const SetVector<Value> &definitions,
-                                         AnalysisState &state) {
-  // Fast path: If READ and WRITE are in 
diff erent regions, their block cannot
-  // be reachable just via unstructured control flow. (Loops due to regions are
-  // covered by `canUseOpDominanceDueToRegions`.)
-  if (uRead->getOwner()->getParentRegion() !=
-      uWrite->getOwner()->getParentRegion())
+                                         OneShotAnalysisState &state) {
+  // No multi-block region means no block-based cycle for dominance to miss.
+  if (!state.mayHaveUnstructuredControlFlow())
     return true;
 
+  assert(!definitions.empty() && "expected at least one definition");
   Block *readBlock = uRead->getOwner()->getBlock();
   Block *writeBlock = uWrite->getOwner()->getBlock();
-  for (Value def : definitions) {
-    Block *defBlock = def.getParentBlock();
-    if (readBlock->isReachable(writeBlock, {defBlock}) &&
-        writeBlock->isReachable(readBlock, {defBlock}))
-      return false;
-  }
 
-  return true;
+  SmallVector<Block *> defBlocks;
+  defBlocks.reserve(definitions.size());
+  for (Value def : definitions)
+    defBlocks.push_back(def.getParentBlock());
+  llvm::sort(defBlocks);
+  defBlocks.erase(llvm::unique(defBlocks), defBlocks.end());
+
+  return state.canUseOpDominanceDueToBlocksCached(
+      readBlock, writeBlock, defBlocks, [&] {
+        return computeCanUseOpDominanceDueToBlocks(uRead, uWrite, definitions,
+                                                   state);
+      });
 }
 
 static bool canUseOpDominance(OpOperand *uRead, OpOperand *uWrite,
                               const SetVector<Value> &definitions,
-                              AnalysisState &state) {
+                              OneShotAnalysisState &state) {
   return canUseOpDominanceDueToRegions(uRead, uWrite, definitions, state) &&
          canUseOpDominanceDueToBlocks(uRead, uWrite, definitions, state);
 }
@@ -658,6 +848,27 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
       continue;
     }
 
+    // If `useDominance` is true below, then for CFG regions we know that we can
+    // rule out blocks containing DEFs of block args for the reachability check
+    // in `cannotHappenAfter`. See the conditions in
+    // `canUseOpDominanceDueToBlocks`. E.g.:
+    //
+    // ^h(%t: tensor<?xf32>):            // DEF
+    //   cf.cond_br %c, ^exit, ^body
+    // ^body:
+    //   %w = "writing_op"(%t)           // WRITE
+    //   cf.br ^h(%w)
+    // ^exit:
+    //   "reading_op"(%t)                // READ
+    //
+    // The only CFG path from WRITE to READ is ^body -> ^h -> ^exit, so READ
+    // cannot happen after WRITE without going through the DEF in ^h, and
+    // therefore there is no conflict.
+    SmallPtrSet<Block *, 16> bbArgDefBlocks;
+    for (Value def : definitions)
+      if (isa<BlockArgument>(def))
+        bbArgDefBlocks.insert(def.getParentBlock());
+
     // Look for conflicting memory writes. Potential conflicts are writes to an
     // alias that have been decided to bufferize inplace.
     for (OpOperand *uConflictingWrite : usesWrite) {
@@ -679,14 +890,15 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
       // Inside of repetitive regions, ops may be executed multiple times and op
       // dominance cannot be used to rule out conflicts.
       if (useDominance) {
-        // No conflict if the readingOp dominates conflictingWritingOp, i.e.,
-        // the write is not visible when reading.
+        // No conflict if the readingOp cannot happen after
+        // conflictingWritingOp, i.e., the write is not visible when reading.
         //
         // Note: If ops are executed multiple times (e.g., because they are
-        //       inside a loop), there may be no meaningful `happensBefore`
+        //       inside a loop), there may be no meaningful `cannotHappenAfter`
         //       relationship.
-        if (happensBefore(readingOp, conflictingWritingOp, domInfo)) {
-          LDBG() << "  no conflict: read happens before write";
+        if (cannotHappenAfter(readingOp, conflictingWritingOp, domInfo, state,
+                              bbArgDefBlocks)) {
+          LDBG() << "  no conflict: read cannot happen after write";
           continue;
         }
 
@@ -762,11 +974,11 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
       for (Value definition : definitions) {
         LDBG() << "  * definition = " << definition;
 
-        // No conflict if the conflicting write happens before the definition.
+        // No conflict if the conflicting write cannot happen after the
+        // definition.
         if (Operation *defOp = definition.getDefiningOp()) {
-          if (happensBefore(conflictingWritingOp, defOp, domInfo)) {
-            // conflictingWritingOp happens before defOp. No conflict.
-            LDBG() << "    no conflict: write happens before definition";
+          if (cannotHappenAfter(conflictingWritingOp, defOp, domInfo, state)) {
+            LDBG() << "    no conflict: write cannot happen after definition";
             continue;
           }
           // No conflict if conflictingWritingOp is contained in defOp.
@@ -775,13 +987,17 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
             continue;
           }
         } else {
-          auto bbArg = cast<BlockArgument>(definition);
-          Block *block = bbArg.getOwner();
-          if (!block->findAncestorOpInBlock(*conflictingWritingOp)) {
-            LDBG() << "    no conflict: definition is bbArg "
-                      "and write happens outside of block";
-            // conflictingWritingOp happens outside of the block. No
-            // conflict.
+          // A bbArg is defined at block entry. The write happens after that
+          // only if it is in the same block or in a reachable block (e.g. a
+          // successor).
+          Block *defBlock = cast<BlockArgument>(definition).getOwner();
+          Operation *writeOp = defBlock->getParent()->findAncestorOpInRegion(
+              *conflictingWritingOp);
+          if (!writeOp ||
+              (writeOp->getBlock() != defBlock &&
+               !state.isReachableCached(defBlock, writeOp->getBlock()))) {
+            LDBG() << "    no conflict: definition is bbArg and write cannot "
+                      "happen after def";
             continue;
           }
         }
@@ -973,6 +1189,22 @@ OneShotAnalysisState::findDefinitionsCached(OpOperand *opOperand) {
   return cachedDefinitions[value];
 }
 
+bool OneShotAnalysisState::isReachableCached(
+    Block *from, Block *to, const SmallPtrSetImpl<Block *> *barriers) {
+  if (!cfgReachabilityCache)
+    cfgReachabilityCache = std::make_unique<CFGReachabilityCache>();
+  return cfgReachabilityCache->isReachable(from, to, barriers);
+}
+
+bool OneShotAnalysisState::canUseOpDominanceDueToBlocksCached(
+    Block *readBlock, Block *writeBlock, ArrayRef<Block *> defBlocks,
+    function_ref<bool()> compute) {
+  if (!opDominanceBlockCache)
+    opDominanceBlockCache = std::make_unique<OpDominanceBlockCache>();
+  return opDominanceBlockCache->getOrCompute(readBlock, writeBlock, defBlocks,
+                                             compute);
+}
+
 bool OneShotAnalysisState::areNonConflictingSubsetsCached(
     OpOperand *uRead, OpOperand *uConflictingWrite) {
   auto key = std::make_pair(uRead, uConflictingWrite);
@@ -986,6 +1218,10 @@ void OneShotAnalysisState::resetCache() {
   AnalysisState::resetCache();
   cachedDefinitions.clear();
   nonConflictingSubsetCache.clear();
+  if (cfgReachabilityCache)
+    cfgReachabilityCache->clear();
+  if (opDominanceBlockCache)
+    opDominanceBlockCache->clear();
 }
 
 /// Determine if `operand` can be bufferized in-place.

diff  --git a/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
index bc8ac8d559aa4..20de1af1b47ae 100644
--- a/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
+++ b/mlir/test/Dialect/ControlFlow/one-shot-bufferize-analysis.mlir
@@ -219,3 +219,717 @@ func.func @looping_branches_sequence_inside_def(%f: f32) {
 ^bb3:
   func.return
 }
+
+// -----
+
+// Sequential loops; loop 2 forwards loop 1's bbarg. Inserts are inplace.
+// CHECK-LABEL: func @sequential_loops_forward_bbarg(
+func.func @sequential_loops_forward_bbarg() -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  %enter = "test.foo"() : () -> (i1)
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+  cf.cond_br %enter, ^bb1(%init : tensor<5xf32>), ^bb1(%init : tensor<5xf32>)
+
+^bb1(%cache: tensor<5xf32>):
+  %c1 = "test.foo"() : () -> (i1)
+  cf.cond_br %c1, ^bb3, ^bb2
+
+^bb2:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %mid = tensor.insert %val into %cache[%pos] : tensor<5xf32>
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %updated1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+  cf.cond_br %enter, ^bb1(%updated1 : tensor<5xf32>), ^bb1(%updated1 : tensor<5xf32>)
+
+^bb3:
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+  cf.cond_br %enter, ^bb4(%cache : tensor<5xf32>), ^bb4(%cache : tensor<5xf32>)
+
+^bb4(%cache2: tensor<5xf32>):
+  %c2 = "test.foo"() : () -> (i1)
+  cf.cond_br %c2, ^bb6, ^bb5
+
+^bb5:
+  %pos3 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %updated2 = tensor.insert %val3 into %cache2[%pos3] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+  cf.cond_br %enter, ^bb4(%updated2 : tensor<5xf32>), ^bb4(%updated2 : tensor<5xf32>)
+
+^bb6:
+  func.return %cache2 : tensor<5xf32>
+}
+
+// -----
+
+// Diamond: read vs write of an OpResult dest. Insert is inplace.
+// CHECK-LABEL: func @diamond_read_write(
+func.func @diamond_read_write(%c: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+  %pos = "test.foo"() : () -> (index)
+// CHECK: tensor.extract
+// CHECK-SAME: __inplace_operands_attr__ = ["true", "none"]
+  %extracted = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%extracted) : (f32) -> ()
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+  cf.br ^bb6(%t : tensor<5xf32>)
+^bb5:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+  cf.br ^bb6(%inserted : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Reads in both arms, write after the join. Insert is inplace.
+// CHECK-LABEL: func @read_then_write_after_join(
+func.func @read_then_write_after_join(%c: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+  %pos = "test.foo"() : () -> (index)
+  %e1 = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e1) : (f32) -> ()
+  cf.br ^bb3
+^bb2:
+  %pos2 = "test.foo"() : () -> (index)
+  %e2 = tensor.extract %t[%pos2] : tensor<5xf32>
+  "test.qux"(%e2) : (f32) -> ()
+  cf.br ^bb3
+^bb3:
+  %pos3 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %t[%pos3] : tensor<5xf32>
+  func.return %inserted : tensor<5xf32>
+}
+
+// -----
+
+// Write in one arm, extract of the original after the join. Insert is out-of-place.
+// CHECK-LABEL: func @write_then_read_after_join(
+func.func @write_then_read_after_join(%c: i1, %f: f32) -> f32 {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %inserted = tensor.insert %val into %t[%pos] : tensor<5xf32>
+  cf.br ^bb3(%inserted : tensor<5xf32>)
+^bb2:
+  cf.br ^bb3(%t : tensor<5xf32>)
+^bb3(%r: tensor<5xf32>):
+  %pos2 = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos2] : tensor<5xf32>
+  func.return %e : f32
+}
+
+// -----
+
+// bb0 -> bb2|bb5, bb2 -> bb5|bb6. Read in bb2, write in bb5. Insert is inplace.
+// CHECK-LABEL: func @read_predecessor_write_successor(
+func.func @read_predecessor_write_successor(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+  %pos = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.cond_br %c2, ^bb5, ^bb6(%t : tensor<5xf32>)
+^bb5:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+  cf.br ^bb6(%inserted : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Same CFG; write in bb2, extract of the original in bb5. Insert is out-of-place.
+// CHECK-LABEL: func @write_predecessor_read_successor(
+func.func @write_predecessor_read_successor(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb2, ^bb5
+^bb2:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %inserted = tensor.insert %val into %t[%pos] : tensor<5xf32>
+  cf.cond_br %c2, ^bb5, ^bb6(%inserted : tensor<5xf32>)
+^bb5:
+  %pos2 = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos2] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^bb6(%t : tensor<5xf32>)
+^bb6(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Three exclusive arms: read, write, passthrough. Insert is inplace.
+// CHECK-LABEL: func @three_way_exclusive_read_write(
+func.func @three_way_exclusive_read_write(%c1: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c1, ^read, ^rest
+^rest:
+  cf.cond_br %c2, ^write, ^pass
+^read:
+  %pos = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^join(%t : tensor<5xf32>)
+^write:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+  cf.br ^join(%inserted : tensor<5xf32>)
+^pass:
+  cf.br ^join(%t : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Both diamond arms write the same dest. Both inserts are inplace.
+// CHECK-LABEL: func @writes_in_both_diamond_arms(
+func.func @writes_in_both_diamond_arms(%c: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^bb1, ^bb2
+^bb1:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %i1 = tensor.insert %val into %t[%pos] : tensor<5xf32>
+  cf.br ^bb3(%i1 : tensor<5xf32>)
+^bb2:
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %i2 = tensor.insert %val2 into %t[%pos2] : tensor<5xf32>
+  cf.br ^bb3(%i2 : tensor<5xf32>)
+^bb3(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Nested diamond: inner read vs outer write. Insert is inplace.
+// CHECK-LABEL: func @nested_diamond_exclusive(
+func.func @nested_diamond_exclusive(%c: i1, %c2: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^outer_read, ^outer_write
+^outer_read:
+  cf.cond_br %c2, ^inner_read, ^inner_pass
+^inner_read:
+  %pos = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^join(%t : tensor<5xf32>)
+^inner_pass:
+  cf.br ^join(%t : tensor<5xf32>)
+^outer_write:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+  cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Diamond inside a loop; dest defined outside. Insert is out-of-place.
+// CHECK-LABEL: func @diamond_inside_loop_outside_def(
+func.func @diamond_inside_loop_outside_def(%c: i1, %back: i1, %f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.br ^header
+^header:
+  cf.cond_br %c, ^read, ^write
+^read:
+  %pos = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^join
+^write:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+  cf.br ^join
+^join:
+  cf.cond_br %back, ^header, ^exit
+^exit:
+  func.return %t : tensor<5xf32>
+}
+
+// -----
+
+// Diamond inside a loop; dest defined in the loop body. Insert is out-of-place.
+// CHECK-LABEL: func @diamond_inside_loop_inside_def(
+func.func @diamond_inside_loop_inside_def(%c: i1, %back: i1, %f: f32) -> tensor<5xf32> {
+  cf.br ^header
+^header:
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.cond_br %c, ^read, ^write
+^read:
+  %pos = "test.foo"() : () -> (index)
+  %e = tensor.extract %t[%pos] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^join(%t : tensor<5xf32>)
+^write:
+  %pos2 = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %inserted = tensor.insert %val into %t[%pos2] : tensor<5xf32>
+  cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  cf.cond_br %back, ^header, ^exit(%r : tensor<5xf32>)
+^exit(%out: tensor<5xf32>):
+  func.return %out : tensor<5xf32>
+}
+
+// -----
+
+// Loop then diamond on the forwarded bbarg. Inserts are inplace.
+// CHECK-LABEL: func @sequential_loop_then_diamond(
+func.func @sequential_loop_then_diamond(%c: i1, %enter: i1) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.cond_br %enter, ^loop(%init : tensor<5xf32>), ^loop(%init : tensor<5xf32>)
+^loop(%cache: tensor<5xf32>):
+  %c1 = "test.foo"() : () -> (i1)
+  cf.cond_br %c1, ^after, ^body
+^body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %mid = tensor.insert %val into %cache[%pos] : tensor<5xf32>
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %updated = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+// CHECK: cf.cond_br
+// CHECK-SAME: {__inplace_operands_attr__ = ["none", "true", "true"]}
+  cf.cond_br %enter, ^loop(%updated : tensor<5xf32>), ^loop(%updated : tensor<5xf32>)
+^after:
+  cf.cond_br %c, ^read(%cache : tensor<5xf32>), ^write(%cache : tensor<5xf32>)
+^read(%r0: tensor<5xf32>):
+  %pos3 = "test.foo"() : () -> (index)
+  %e = tensor.extract %r0[%pos3] : tensor<5xf32>
+  "test.qux"(%e) : (f32) -> ()
+  cf.br ^join(%r0 : tensor<5xf32>)
+^write(%r1: tensor<5xf32>):
+  %pos4 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val3 into %r1[%pos4] : tensor<5xf32>
+  cf.br ^join(%inserted : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Sequential inner loops inside an outer loop. There is no actual RaW
+// conflict: the only CFG path from the l2 insert to the later l1 insert is
+// the outer-loop back-edge, which is a 
diff erent outer iteration. The
+// analysis is currently conservative with regards to that back-edge, so the
+// first insert dest is out-of-place.
+// CHECK-LABEL: func @outer_loop_sequential_inner_loops(
+func.func @outer_loop_sequential_inner_loops(%again: i1) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+  cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+  %e1 = "test.foo"() : () -> (i1)
+  cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+  // No actual conflict; conservative on the outer-loop back-edge.
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+  cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+  cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+  %e2 = "test.foo"() : () -> (i1)
+  cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+  %pos3 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+  cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+  cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+  func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// Sequential inner loops inside an outer loop; dest defined outside. The
+// outer backedge lets l1 write and l2 read the same outside value on
+// 
diff erent iterations, so the insert dest is out-of-place.
+// CHECK-LABEL: func @outer_loop_seq_inners_outside_def(
+func.func @outer_loop_seq_inners_outside_def(%c: i1, %back: i1, %f: f32) -> f32 {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  cf.br ^outer
+^outer:
+  cf.br ^l1
+^l1:
+  %e1 = "test.foo"() : () -> (i1)
+  cf.cond_br %e1, ^l2, ^l1body
+^l1body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %mid = tensor.insert %val into %t[%pos] : tensor<5xf32>
+  cf.br ^l1
+^l2:
+  %e2 = "test.foo"() : () -> (i1)
+  cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+  %pos2 = "test.foo"() : () -> (index)
+  %ex = tensor.extract %t[%pos2] : tensor<5xf32>
+  "test.qux"(%ex) : (f32) -> ()
+  cf.br ^l2
+^outer_end:
+  cf.cond_br %back, ^outer, ^exit
+^exit:
+  %pos3 = "test.foo"() : () -> (index)
+  %r = tensor.extract %t[%pos3] : tensor<5xf32>
+  func.return %r : f32
+}
+
+// -----
+
+// l1 writes the header bbarg then extracts it. The extract must observe %c1
+// before the insert, so the insert dest is out-of-place.
+// CHECK-LABEL: func @l1_write_then_extract_bbarg(
+func.func @l1_write_then_extract_bbarg(%again: i1) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+  cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+  %e1 = "test.foo"() : () -> (i1)
+  cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %upd1 = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+  %posr = "test.foo"() : () -> (index)
+  %old = tensor.extract %c1[%posr] : tensor<5xf32>
+  "test.qux"(%old) : (f32) -> ()
+  cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+  cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+  %e2 = "test.foo"() : () -> (i1)
+  cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+  %pos3 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+  cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+  cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+  func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// Extra edge l2body -> l1body, not through the outer header. The tensor is
+// copied on the branches into the inner loops.
+// CHECK-LABEL: func @irreducible_l2_to_l1(
+func.func @irreducible_l2_to_l1(%again: i1, %weird: i1) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["true"]}
+  cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+// CHECK: cf.br
+// CHECK-SAME: {__inplace_operands_attr__ = ["false"]}
+  cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+  %e1 = "test.foo"() : () -> (i1)
+  cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+  cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+  cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+  %e2 = "test.foo"() : () -> (i1)
+  cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+  %pos3 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+  cf.cond_br %weird, ^l1body, ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+  cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+  func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// l1 writes the carried dest then extracts the outer bbarg. The extract must
+// observe %cache before the first write, so that insert dest is out-of-place.
+// The chained insert writes the copy and is inplace.
+// CHECK-LABEL: func @l1_write_then_extract_carried(
+func.func @l1_write_then_extract_carried(%again: i1) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.br ^outer(%init : tensor<5xf32>)
+^outer(%cache: tensor<5xf32>):
+  cf.br ^l1(%cache : tensor<5xf32>)
+^l1(%c1: tensor<5xf32>):
+  %e1 = "test.foo"() : () -> (i1)
+  cf.cond_br %e1, ^l2entry, ^l1body
+^l1body:
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %mid = tensor.insert %val into %c1[%pos] : tensor<5xf32>
+  %pos2 = "test.foo"() : () -> (index)
+  %val2 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd1 = tensor.insert %val2 into %mid[%pos2] : tensor<5xf32>
+  %posr = "test.foo"() : () -> (index)
+  %old = tensor.extract %cache[%posr] : tensor<5xf32>
+  "test.qux"(%old) : (f32) -> ()
+  cf.br ^l1(%upd1 : tensor<5xf32>)
+^l2entry:
+  cf.br ^l2(%c1 : tensor<5xf32>)
+^l2(%c2: tensor<5xf32>):
+  %e2 = "test.foo"() : () -> (i1)
+  cf.cond_br %e2, ^outer_end, ^l2body
+^l2body:
+  %pos3 = "test.foo"() : () -> (index)
+  %val3 = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %upd2 = tensor.insert %val3 into %c2[%pos3] : tensor<5xf32>
+  cf.br ^l2(%upd2 : tensor<5xf32>)
+^outer_end:
+  cf.cond_br %again, ^outer(%c2 : tensor<5xf32>), ^exit
+^exit:
+  func.return %c2 : tensor<5xf32>
+}
+
+// -----
+
+// Mutually exclusive cases each take a tensor bbArg. Only one is live at
+// runtime. That does not put every case block into the RaW extra-barrier set:
+// findDefinitions stops at a bbArg, and sibling case bbArgs are not in scope
+// together, so they cannot all be definitions of one read.
+// Passing the original dest to the join after the insert is a RaW, so the
+// insert dest is out-of-place.
+// CHECK-LABEL: func @switch_write_pass_original(
+func.func @switch_write_pass_original(%flag: i32) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.switch %flag : i32, [
+    default: ^c0(%init : tensor<5xf32>),
+    1: ^c1(%init : tensor<5xf32>)
+  ]
+^c0(%a0: tensor<5xf32>):
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+  %inserted = tensor.insert %val into %a0[%pos] : tensor<5xf32>
+  cf.br ^join(%a0 : tensor<5xf32>)
+^c1(%a1: tensor<5xf32>):
+  cf.br ^join(%a1 : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Same switch shape; the join is the insert result, so there is no RaW and
+// the insert dest is inplace.
+// CHECK-LABEL: func @switch_write_pass_result(
+func.func @switch_write_pass_result(%flag: i32) -> tensor<5xf32> {
+  %init = tensor.empty() : tensor<5xf32>
+  cf.switch %flag : i32, [
+    default: ^c0(%init : tensor<5xf32>),
+    1: ^c1(%init : tensor<5xf32>)
+  ]
+^c0(%a0: tensor<5xf32>):
+  %pos = "test.foo"() : () -> (index)
+  %val = "test.bar"() : () -> (f32)
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "true", "none"]
+  %inserted = tensor.insert %val into %a0[%pos] : tensor<5xf32>
+  cf.br ^join(%inserted : tensor<5xf32>)
+^c1(%a1: tensor<5xf32>):
+  cf.br ^join(%a1 : tensor<5xf32>)
+^join(%r: tensor<5xf32>):
+  func.return %r : tensor<5xf32>
+}
+
+// -----
+
+// Nested diamond in execute_region, called from an outer CFG loop. Dest is
+// defined outside the loop, so the insert is out-of-place.
+// CHECK-LABEL: func @nested_diamond_in_outer_cfg_loop(
+func.func @nested_diamond_in_outer_cfg_loop(%f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  cf.br ^loop(%c0 : index)
+^loop(%i: index):
+  %take_read = arith.cmpi eq, %i, %c1 : index
+  %r = scf.execute_region -> tensor<5xf32> {
+    cf.cond_br %take_read, ^read, ^write
+  ^read:
+    %e = tensor.extract %t[%c0] : tensor<5xf32>
+    "test.qux"(%e) : (f32) -> ()
+    cf.br ^join(%t : tensor<5xf32>)
+  ^write:
+    %val = "test.bar"() : () -> f32
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+    %inserted = tensor.insert %val into %t[%c0] : tensor<5xf32>
+    cf.br ^join(%inserted : tensor<5xf32>)
+  ^join(%out: tensor<5xf32>):
+    scf.yield %out : tensor<5xf32>
+  }
+  %next = arith.addi %i, %c1 : index
+  %again = arith.cmpi ult, %next, %c2 : index
+  cf.cond_br %again, ^loop(%next : index), ^exit(%r : tensor<5xf32>)
+^exit(%result: tensor<5xf32>):
+  func.return %result : tensor<5xf32>
+}
+
+// -----
+
+// Same outer CFG loop, but read and write are in separate execute_region ops.
+// Insert is still out-of-place.
+// CHECK-LABEL: func @two_execute_regions_in_outer_cfg_loop(
+func.func @two_execute_regions_in_outer_cfg_loop(%f: f32) -> tensor<5xf32> {
+  %empty = tensor.empty() : tensor<5xf32>
+  %t = linalg.fill ins(%f : f32) outs(%empty : tensor<5xf32>) -> tensor<5xf32>
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  cf.br ^loop(%c0 : index)
+^loop(%i: index):
+  %take_read = arith.cmpi eq, %i, %c1 : index
+  scf.execute_region {
+    cf.cond_br %take_read, ^do_read, ^skip_read
+  ^do_read:
+    %e = tensor.extract %t[%c0] : tensor<5xf32>
+    "test.qux"(%e) : (f32) -> ()
+    cf.br ^join_read
+  ^skip_read:
+    cf.br ^join_read
+  ^join_read:
+    scf.yield
+  }
+  %r = scf.execute_region -> tensor<5xf32> {
+    cf.cond_br %take_read, ^skip_write, ^do_write
+  ^do_write:
+    %val = "test.bar"() : () -> f32
+// CHECK: tensor.insert
+// CHECK-SAME: __inplace_operands_attr__ = ["none", "false", "none"]
+    %inserted = tensor.insert %val into %t[%c0] : tensor<5xf32>
+    cf.br ^join_write(%inserted : tensor<5xf32>)
+  ^skip_write:
+    cf.br ^join_write(%t : tensor<5xf32>)
+  ^join_write(%w: tensor<5xf32>):
+    scf.yield %w : tensor<5xf32>
+  }
+  %next = arith.addi %i, %c1 : index
+  %again = arith.cmpi ult, %next, %c2 : index
+  cf.cond_br %again, ^loop(%next : index), ^exit(%r : tensor<5xf32>)
+^exit(%result: tensor<5xf32>):
+  func.return %result : tensor<5xf32>
+}