[Mlir-commits] [mlir] [mlir][linalg] Hoist masked vector transfer pairs (PR #221481)
Hugo Pompougnac
llvmlistbot at llvm.org
Wed Sep 9 01:56:18 PDT 2026
https://github.com/qaco updated https://github.com/llvm/llvm-project/pull/221481
>From e6b8fdc93c4ed86ce8389fb123e9e159b06cfdf5 Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Tue, 1 Sep 2026 15:05:55 +0200
Subject: [PATCH 01/10] [mlir][linalg] Hoist transfer pairs through disjoint
subview slices
---
.../Dialect/Linalg/Transforms/Hoisting.cpp | 128 +++++++++++-
mlir/test/Dialect/Linalg/hoisting.mlir | 192 ++++++++++++++++++
2 files changed, 318 insertions(+), 2 deletions(-)
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index a573b4a54dbba..30a5047685fff 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -17,11 +17,14 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/Utils.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/SCF/Utils/Utils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Dominance.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "llvm/Support/Debug.h"
@@ -195,6 +198,116 @@ static bool noAliasingUseInLoop(vector::TransferReadOp transferRead,
return true;
}
+namespace {
+/// Static footprint of a memref value resolved through a chain of
+/// rank-preserving, unit-stride `memref.subview` ops: the underlying root
+/// memref and, per root dimension, the `[offset, offset + size)` interval it
+/// covers.
+struct SubviewFootprint {
+ Value root;
+ SmallVector<int64_t> offsets;
+ SmallVector<int64_t> sizes;
+};
+} // namespace
+
+/// Resolve `base` through a `memref.subview` chain to its static footprint, or
+/// `std::nullopt` if it cannot be modelled statically.
+static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
+ SmallVector<memref::SubViewOp> chain;
+ Value source = base;
+ while (auto sv = source.getDefiningOp<memref::SubViewOp>()) {
+ chain.push_back(sv);
+ source = sv.getSource();
+ }
+ // The resolved root must be a genuine buffer, not another kind of view.
+ if (isa_and_nonnull<ViewLikeOpInterface>(source.getDefiningOp()))
+ return std::nullopt;
+
+ auto rootTy = dyn_cast<MemRefType>(source.getType());
+ if (!rootTy || !rootTy.hasStaticShape())
+ return std::nullopt;
+ SubviewFootprint fp;
+ fp.root = source;
+ // Seed with the whole-root footprint (offset 0, full extent per dimension):
+ // this is the result for a bare buffer, and each subview in the chain narrows
+ // it below.
+ fp.offsets.assign(rootTy.getRank(), 0);
+ fp.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
+ // Compose from the outermost subview (closest to the root) inward.
+ for (memref::SubViewOp sv : llvm::reverse(chain)) {
+ // Rank-reducing subviews would misalign the per-dimension bookkeeping.
+ if (sv.getSourceType().getRank() != sv.getType().getRank())
+ return std::nullopt;
+ ArrayRef<int64_t> offs = sv.getStaticOffsets();
+ ArrayRef<int64_t> szs = sv.getStaticSizes();
+ ArrayRef<int64_t> strides = sv.getStaticStrides();
+ if (llvm::any_of(offs, ShapedType::isDynamic) ||
+ llvm::any_of(szs, ShapedType::isDynamic) ||
+ llvm::any_of(strides, [](int64_t s) { return s != 1; }))
+ return std::nullopt;
+ for (unsigned d = 0, e = offs.size(); d < e; ++d) {
+ fp.offsets[d] += offs[d];
+ fp.sizes[d] = szs[d];
+ }
+ }
+ return fp;
+}
+
+/// True if two footprints provably cover disjoint memory: they must share the
+/// same root and be separated along at least one dimension.
+static bool footprintsDisjoint(const SubviewFootprint &a,
+ const SubviewFootprint &b) {
+ if (a.root != b.root)
+ return false;
+ for (unsigned d = 0, e = a.offsets.size(); d < e; ++d) {
+ int64_t aHi = a.offsets[d] + a.sizes[d];
+ int64_t bHi = b.offsets[d] + b.sizes[d];
+ if (aHi <= b.offsets[d] || bHi <= a.offsets[d])
+ return true;
+ }
+ return false;
+}
+
+/// Subview-aware safety check for hoisting `read`/`write` out of `loop`.
+static bool footprintAliasingIsSafe(LoopLikeOpInterface loop,
+ vector::TransferReadOp read,
+ vector::TransferWriteOp write) {
+ std::optional<SubviewFootprint> ourFp =
+ resolveSubviewFootprint(read.getBase());
+ if (!ourFp)
+ return false;
+ // `resolveSubviewFootprint` guarantees the root is a genuine buffer.
+ Value ourBuffer = ourFp->root;
+ auto reachesOurBuffer = [&](Value operand) {
+ auto memrefOperand = dyn_cast<MemrefValue>(operand);
+ return memrefOperand && memref::skipViewLikeOps(memrefOperand) == ourBuffer;
+ };
+ WalkResult res = loop->walk([&](Operation *op) {
+ if (op == read.getOperation() || op == write.getOperation())
+ return WalkResult::advance();
+ if (auto xfer = dyn_cast<VectorTransferOpInterface>(op)) {
+ Value otherBase = xfer.getBase();
+ // A tensor transfer or a different underlying buffer cannot alias ours.
+ if (!reachesOurBuffer(otherBase))
+ return WalkResult::advance();
+ // Same buffer: only safe if we can prove the accessed slices are
+ // statically disjoint.
+ std::optional<SubviewFootprint> otherFp =
+ resolveSubviewFootprint(otherBase);
+ if (otherFp && footprintsDisjoint(*ourFp, *otherFp))
+ return WalkResult::advance();
+ return WalkResult::interrupt();
+ }
+ // Any other op that may touch memory aliases ours if one of its memref
+ // operands reaches our buffer.
+ if (!isMemoryEffectFree(op) &&
+ llvm::any_of(op->getOperands(), reachesOurBuffer))
+ return WalkResult::interrupt();
+ return WalkResult::advance();
+ });
+ return !res.wasInterrupted();
+}
+
void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
bool verifyNonZeroTrip) {
bool changed = true;
@@ -314,6 +427,15 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
// Check 2. Note, since both xfer Ops share the source, we only need to
// look at one of them.
auto base = transferRead.getBase();
+ // Whether hoisting is safe despite a view base. Computed lazily and
+ // cached since it is only consulted when a view is present.
+ std::optional<bool> viewAliasingIsSafeCache;
+ auto viewAliasingIsSafe = [&]() {
+ if (!viewAliasingIsSafeCache)
+ viewAliasingIsSafeCache =
+ footprintAliasingIsSafe(loop, transferRead, transferWrite);
+ return *viewAliasingIsSafeCache;
+ };
auto *source = base.getDefiningOp();
if (source) {
// NOTE: We treat `memref.assume_alignment` as a special case.
@@ -337,11 +459,13 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
if (numInLoopUses && memPreAlignment.hasOneUse())
source = memPreAlignment.getDefiningOp();
}
- if (isa_and_nonnull<ViewLikeOpInterface>(source))
+ if (isa_and_nonnull<ViewLikeOpInterface>(source) &&
+ !viewAliasingIsSafe())
return WalkResult::advance();
}
- if (llvm::any_of(base.getUsers(), llvm::IsaPred<ViewLikeOpInterface>))
+ if (llvm::any_of(base.getUsers(), llvm::IsaPred<ViewLikeOpInterface>) &&
+ !viewAliasingIsSafe())
return WalkResult::advance();
// Check 3.
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index 457449f92f92f..ed87a9ff1d062 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -1171,3 +1171,195 @@ module attributes {transform.with_named_sequence} {
transform.yield
}
}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for hoisting through `memref.subview` views, guarded by a static
+/// footprint disjointness check.
+///----------------------------------------------------------------------------------------
+
+// Disjoint static subviews of one buffer: both transfer pairs are hoisted.
+
+// CHECK-LABEL: func.func @hoist_disjoint_static_subviews(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4xf32>,
+// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0] [2] [1]
+// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][2] [2] [1]
+// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
+// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
+// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2xf32>, vector<2xf32>) {
+// CHECK-NEXT: %[[U0:.*]] = "val_use"(%[[I0]])
+// CHECK-NEXT: %[[U1:.*]] = "val_use"(%[[I1]])
+// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
+// CHECK-NEXT: }
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
+func.func @hoist_disjoint_static_subviews(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ %sv1 = memref.subview %mem[2][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 2>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 2>>, vector<2xf32>
+ %u1 = "val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 2>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Overlapping static subviews are not provably disjoint: hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_overlapping_static_subviews(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_overlapping_static_subviews(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ %sv1 = memref.subview %mem[1][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 1>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 1>>, vector<2xf32>
+ %u1 = "val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 1>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A dynamic subview offset yields no static footprint: hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_dynamic_offset_subview(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_dynamic_offset_subview(
+ %mem: memref<4xf32>, %off: index, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv = memref.subview %mem[%off][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: ?>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Subview rooted at a collapse_shape (not a genuine buffer): hoisting blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_subview_of_collapse_shape(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_subview_of_collapse_shape(
+ %mem: memref<2x2xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %collapse = memref.collapse_shape %mem [[0, 1]] : memref<2x2xf32> into memref<4xf32>
+ %sv = memref.subview %collapse[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A non-transfer op touching the buffer may alias: hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_subview_extra_memory_effect(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: "prevent_hoisting"
+// CHECK: }
+
+func.func @negative_hoist_subview_extra_memory_effect(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ "prevent_hoisting"(%mem) : (memref<4xf32>) -> ()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
>From d4e567959d87f6701cfd0e93e93b78ed503a3c96 Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Wed, 2 Sep 2026 12:03:15 +0200
Subject: [PATCH 02/10] Extract hasNoAliasingAccessInScope in MemRefUtils
---
.../mlir/Dialect/MemRef/Utils/MemRefUtils.h | 14 ++
.../Dialect/Linalg/Transforms/Hoisting.cpp | 165 ++----------------
mlir/lib/Dialect/MemRef/Utils/CMakeLists.txt | 1 +
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 116 ++++++++++++
mlir/test/Dialect/Linalg/hoisting.mlir | 29 +--
5 files changed, 164 insertions(+), 161 deletions(-)
diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index c93da6ebada7c..b40919c621b78 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -198,6 +198,20 @@ LogicalResult resolveSourceIndicesRankReducingSubview(
/// negative.
bool hasNegativeStaticStride(MemRefType memRefTy);
+/// Return "true" when no other access nested in `scope` can alias `base`, so
+/// relocating it is safe with respect to aliasing.
+///
+/// `getAccessedMemref` maps an op to the memref it accesses (null if
+/// unmodelled). Visiting the in-`scope` users of `base`'s underlying buffer, a
+/// user conflicts unless it is effect-free, in `excludedOps`, touches a slice
+/// disjoint from `base`'s static footprint, or only reads when `readsAreSafe`
+/// is true. `excludedOps` are left out of the analysis, typically because the
+/// caller intends to relocate them.
+bool hasNoAliasingAccessInScope(
+ Value base, Operation *scope,
+ function_ref<Value(Operation *)> getAccessedMemref,
+ ArrayRef<Operation *> excludedOps = {}, bool readsAreSafe = false);
+
} // namespace memref
} // namespace mlir
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 30a5047685fff..18101e51ff6bb 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -24,7 +24,6 @@
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/IR/Dominance.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "llvm/Support/Debug.h"
@@ -164,148 +163,11 @@ void mlir::linalg::hoistRedundantVectorBroadcasts(RewriterBase &rewriter,
}
}
-static bool noAliasingUseInLoop(vector::TransferReadOp transferRead,
- LoopLikeOpInterface loop) {
- Value source = transferRead.getBase();
-
- // Skip view-like Ops and retrive the actual soruce Operation
- while (auto viewLike = source.getDefiningOp<ViewLikeOpInterface>()) {
- if (viewLike.getViewDest() != source) {
- break;
- }
- source = viewLike.getViewSource();
- }
-
- llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
- source.getUsers().end());
- llvm::SmallDenseSet<Operation *, 32> processed;
- while (!users.empty()) {
- Operation *user = users.pop_back_val();
- // If the user has already been processed skip.
- if (!processed.insert(user).second)
- continue;
- if (auto viewLike = dyn_cast<ViewLikeOpInterface>(user)) {
- Value viewDest = viewLike.getViewDest();
- users.append(viewDest.getUsers().begin(), viewDest.getUsers().end());
- continue;
- }
- if (isMemoryEffectFree(user) || isa<vector::TransferReadOp>(user))
- continue;
- if (!loop->isAncestor(user))
- continue;
- return false;
- }
- return true;
-}
-
-namespace {
-/// Static footprint of a memref value resolved through a chain of
-/// rank-preserving, unit-stride `memref.subview` ops: the underlying root
-/// memref and, per root dimension, the `[offset, offset + size)` interval it
-/// covers.
-struct SubviewFootprint {
- Value root;
- SmallVector<int64_t> offsets;
- SmallVector<int64_t> sizes;
-};
-} // namespace
-
-/// Resolve `base` through a `memref.subview` chain to its static footprint, or
-/// `std::nullopt` if it cannot be modelled statically.
-static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
- SmallVector<memref::SubViewOp> chain;
- Value source = base;
- while (auto sv = source.getDefiningOp<memref::SubViewOp>()) {
- chain.push_back(sv);
- source = sv.getSource();
- }
- // The resolved root must be a genuine buffer, not another kind of view.
- if (isa_and_nonnull<ViewLikeOpInterface>(source.getDefiningOp()))
- return std::nullopt;
-
- auto rootTy = dyn_cast<MemRefType>(source.getType());
- if (!rootTy || !rootTy.hasStaticShape())
- return std::nullopt;
- SubviewFootprint fp;
- fp.root = source;
- // Seed with the whole-root footprint (offset 0, full extent per dimension):
- // this is the result for a bare buffer, and each subview in the chain narrows
- // it below.
- fp.offsets.assign(rootTy.getRank(), 0);
- fp.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
- // Compose from the outermost subview (closest to the root) inward.
- for (memref::SubViewOp sv : llvm::reverse(chain)) {
- // Rank-reducing subviews would misalign the per-dimension bookkeeping.
- if (sv.getSourceType().getRank() != sv.getType().getRank())
- return std::nullopt;
- ArrayRef<int64_t> offs = sv.getStaticOffsets();
- ArrayRef<int64_t> szs = sv.getStaticSizes();
- ArrayRef<int64_t> strides = sv.getStaticStrides();
- if (llvm::any_of(offs, ShapedType::isDynamic) ||
- llvm::any_of(szs, ShapedType::isDynamic) ||
- llvm::any_of(strides, [](int64_t s) { return s != 1; }))
- return std::nullopt;
- for (unsigned d = 0, e = offs.size(); d < e; ++d) {
- fp.offsets[d] += offs[d];
- fp.sizes[d] = szs[d];
- }
- }
- return fp;
-}
-
-/// True if two footprints provably cover disjoint memory: they must share the
-/// same root and be separated along at least one dimension.
-static bool footprintsDisjoint(const SubviewFootprint &a,
- const SubviewFootprint &b) {
- if (a.root != b.root)
- return false;
- for (unsigned d = 0, e = a.offsets.size(); d < e; ++d) {
- int64_t aHi = a.offsets[d] + a.sizes[d];
- int64_t bHi = b.offsets[d] + b.sizes[d];
- if (aHi <= b.offsets[d] || bHi <= a.offsets[d])
- return true;
- }
- return false;
-}
-
-/// Subview-aware safety check for hoisting `read`/`write` out of `loop`.
-static bool footprintAliasingIsSafe(LoopLikeOpInterface loop,
- vector::TransferReadOp read,
- vector::TransferWriteOp write) {
- std::optional<SubviewFootprint> ourFp =
- resolveSubviewFootprint(read.getBase());
- if (!ourFp)
- return false;
- // `resolveSubviewFootprint` guarantees the root is a genuine buffer.
- Value ourBuffer = ourFp->root;
- auto reachesOurBuffer = [&](Value operand) {
- auto memrefOperand = dyn_cast<MemrefValue>(operand);
- return memrefOperand && memref::skipViewLikeOps(memrefOperand) == ourBuffer;
- };
- WalkResult res = loop->walk([&](Operation *op) {
- if (op == read.getOperation() || op == write.getOperation())
- return WalkResult::advance();
- if (auto xfer = dyn_cast<VectorTransferOpInterface>(op)) {
- Value otherBase = xfer.getBase();
- // A tensor transfer or a different underlying buffer cannot alias ours.
- if (!reachesOurBuffer(otherBase))
- return WalkResult::advance();
- // Same buffer: only safe if we can prove the accessed slices are
- // statically disjoint.
- std::optional<SubviewFootprint> otherFp =
- resolveSubviewFootprint(otherBase);
- if (otherFp && footprintsDisjoint(*ourFp, *otherFp))
- return WalkResult::advance();
- return WalkResult::interrupt();
- }
- // Any other op that may touch memory aliases ours if one of its memref
- // operands reaches our buffer.
- if (!isMemoryEffectFree(op) &&
- llvm::any_of(op->getOperands(), reachesOurBuffer))
- return WalkResult::interrupt();
- return WalkResult::advance();
- });
- return !res.wasInterrupted();
+/// The memref a vector transfer accesses (a null `Value` for any other op).
+static Value vectorTransferBase(Operation *op) {
+ if (auto xfer = dyn_cast<VectorTransferOpInterface>(op))
+ return xfer.getBase();
+ return {};
}
void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
@@ -399,9 +261,12 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
// Only hoist transfer_read / transfer_write pairs and singleton
// transfer_reads for now.
if (!transferWrite) {
- // Make sure there are no other accesses to the memref before
- // hoisting transfer_read.
- if (noAliasingUseInLoop(transferRead, loop))
+ // Hoisting a lone read is safe as long as no aliasing write remains in
+ // the loop; other reads never conflict.
+ if (memref::hasNoAliasingAccessInScope(transferRead.getBase(), loop,
+ vectorTransferBase,
+ /*excludedOps=*/{},
+ /*readsAreSafe=*/true))
loop.moveOutOfLoop(transferRead);
return WalkResult::advance();
}
@@ -431,9 +296,11 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
// cached since it is only consulted when a view is present.
std::optional<bool> viewAliasingIsSafeCache;
auto viewAliasingIsSafe = [&]() {
- if (!viewAliasingIsSafeCache)
- viewAliasingIsSafeCache =
- footprintAliasingIsSafe(loop, transferRead, transferWrite);
+ if (!viewAliasingIsSafeCache) {
+ Operation *hoistedPair[] = {transferRead, transferWrite};
+ viewAliasingIsSafeCache = memref::hasNoAliasingAccessInScope(
+ base, loop, vectorTransferBase, hoistedPair);
+ }
return *viewAliasingIsSafeCache;
};
auto *source = base.getDefiningOp();
diff --git a/mlir/lib/Dialect/MemRef/Utils/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Utils/CMakeLists.txt
index f8c9f8a7cf3d2..da7c365aa4a0b 100644
--- a/mlir/lib/Dialect/MemRef/Utils/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/Utils/CMakeLists.txt
@@ -8,4 +8,5 @@ add_mlir_dialect_library(MLIRMemRefUtils
MLIRIR
MLIRAffineDialect
MLIRArithUtils
+ MLIRSideEffectInterfaces
)
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index bc019d601dcd9..6859e4c5e8b7c 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -14,6 +14,7 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Interfaces/ViewLikeInterface.h"
#include "llvm/ADT/STLExtras.h"
@@ -351,5 +352,120 @@ bool hasNegativeStaticStride(MemRefType memRefTy) {
});
}
+namespace {
+/// Static footprint of a memref value resolved through a chain of
+/// rank-preserving, unit-stride `memref.subview` ops: the underlying root
+/// memref and, per root dimension, the `[offset, offset + size)` interval it
+/// covers.
+struct SubviewFootprint {
+ Value root;
+ SmallVector<int64_t> offsets;
+ SmallVector<int64_t> sizes;
+};
+} // namespace
+
+/// Resolve `base` through a `memref.subview` chain to its static footprint, or
+/// `std::nullopt` if it cannot be modelled statically.
+static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
+ SmallVector<SubViewOp> chain;
+ Value source = base;
+ while (auto sv = source.getDefiningOp<SubViewOp>()) {
+ chain.push_back(sv);
+ source = sv.getSource();
+ }
+ // The resolved root must be a genuine buffer, not another kind of view.
+ if (isa_and_nonnull<ViewLikeOpInterface>(source.getDefiningOp()))
+ return std::nullopt;
+
+ auto rootTy = dyn_cast<MemRefType>(source.getType());
+ if (!rootTy || !rootTy.hasStaticShape())
+ return std::nullopt;
+ SubviewFootprint fp;
+ fp.root = source;
+ // Seed with the whole-root footprint (offset 0, full extent per dimension).
+ // Each subview in the chain narrows it below.
+ fp.offsets.assign(rootTy.getRank(), 0);
+ fp.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
+ // Compose from the outermost subview (closest to the root) inward.
+ for (SubViewOp sv : llvm::reverse(chain)) {
+ // Rank-reducing subviews would misalign the per-dimension bookkeeping.
+ if (sv.getSourceType().getRank() != sv.getType().getRank())
+ return std::nullopt;
+ ArrayRef<int64_t> offs = sv.getStaticOffsets();
+ ArrayRef<int64_t> szs = sv.getStaticSizes();
+ ArrayRef<int64_t> strides = sv.getStaticStrides();
+ if (llvm::any_of(offs, ShapedType::isDynamic) ||
+ llvm::any_of(szs, ShapedType::isDynamic) ||
+ llvm::any_of(strides, [](int64_t s) { return s != 1; }))
+ return std::nullopt;
+ for (unsigned d = 0, e = offs.size(); d < e; ++d) {
+ fp.offsets[d] += offs[d];
+ fp.sizes[d] = szs[d];
+ }
+ }
+ return fp;
+}
+
+/// True if two footprints provably cover disjoint memory: they must share the
+/// same root and be separated along at least one dimension.
+static bool footprintsDisjoint(const SubviewFootprint &a,
+ const SubviewFootprint &b) {
+ if (a.root != b.root)
+ return false;
+ for (unsigned d = 0, e = a.offsets.size(); d < e; ++d) {
+ int64_t aHi = a.offsets[d] + a.sizes[d];
+ int64_t bHi = b.offsets[d] + b.sizes[d];
+ if (aHi <= b.offsets[d] || bHi <= a.offsets[d])
+ return true;
+ }
+ return false;
+}
+
+bool hasNoAliasingAccessInScope(
+ Value base, Operation *scope,
+ function_ref<Value(Operation *)> getAccessedMemref,
+ ArrayRef<Operation *> excludedOps, bool readsAreSafe) {
+ auto baseMemref = dyn_cast<MemrefValue>(base);
+ if (!baseMemref)
+ return false;
+ Value buffer = skipViewLikeOps(baseMemref);
+ // Static footprint of `base`, if modellable; enables the disjointness escape.
+ std::optional<SubviewFootprint> baseFp = resolveSubviewFootprint(base);
+
+ // Visit the transitive users of the buffer, following views.
+ SmallVector<Operation *> worklist(buffer.getUsers().begin(),
+ buffer.getUsers().end());
+ SmallPtrSet<Operation *, 8> processed;
+ while (!worklist.empty()) {
+ Operation *user = worklist.pop_back_val();
+ if (!processed.insert(user).second)
+ continue;
+ // Views do not access memory; follow them to their own users.
+ if (auto viewLike = dyn_cast<ViewLikeOpInterface>(user)) {
+ Value viewDest = viewLike.getViewDest();
+ worklist.append(viewDest.getUsers().begin(), viewDest.getUsers().end());
+ continue;
+ }
+ if (isMemoryEffectFree(user) || llvm::is_contained(excludedOps, user))
+ continue;
+ if (!scope->isAncestor(user))
+ continue;
+ // In-`scope`, memory-effecting access to the buffer: it conflicts unless it
+ // is a provably-disjoint slice or (when allowed) a read.
+ Value accessed = getAccessedMemref(user);
+ if (baseFp && accessed) {
+ std::optional<SubviewFootprint> userFp =
+ resolveSubviewFootprint(accessed);
+ if (userFp && footprintsDisjoint(*baseFp, *userFp))
+ continue;
+ }
+ if (readsAreSafe && accessed &&
+ !hasEffect<MemoryEffects::Write>(user, accessed))
+ continue;
+ return false;
+ }
+ return true;
+}
+
} // namespace memref
} // namespace mlir
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index ed87a9ff1d062..eadc4af8ba514 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -1264,15 +1264,17 @@ module attributes {transform.with_named_sequence} {
// -----
-// A dynamic subview offset yields no static footprint: hoisting is blocked.
+// A dynamic (but loop-invariant) subview offset with no other access to the
+// buffer: the pair still hoists, since nothing else can alias it.
-// CHECK-LABEL: func.func @negative_hoist_dynamic_offset_subview(
-// CHECK: scf.for {{.*}} step %{{.*}} {
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
+// CHECK-LABEL: func.func @hoist_dynamic_offset_subview(
+// CHECK: %[[SV:.*]] = memref.subview
+// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
+// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
// CHECK: }
+// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
-func.func @negative_hoist_dynamic_offset_subview(
+func.func @hoist_dynamic_offset_subview(
%mem: memref<4xf32>, %off: index, %lb : index, %ub : index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
@@ -1297,15 +1299,18 @@ module attributes {transform.with_named_sequence} {
// -----
-// Subview rooted at a collapse_shape (not a genuine buffer): hoisting blocked.
+// A subview rooted at a collapse_shape that is the buffer's sole accessor: the
+// pair still hoists, since nothing else can alias it.
-// CHECK-LABEL: func.func @negative_hoist_subview_of_collapse_shape(
-// CHECK: scf.for {{.*}} step %{{.*}} {
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
+// CHECK-LABEL: func.func @hoist_subview_of_collapse_shape(
+// CHECK: %[[COLLAPSE:.*]] = memref.collapse_shape
+// CHECK: %[[SV:.*]] = memref.subview %[[COLLAPSE]]
+// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
+// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
// CHECK: }
+// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
-func.func @negative_hoist_subview_of_collapse_shape(
+func.func @hoist_subview_of_collapse_shape(
%mem: memref<2x2xf32>, %lb : index, %ub : index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
>From b1d726ed7d1f2b44e0c1285c39ee62fa5154ba2e Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Wed, 2 Sep 2026 12:13:46 +0200
Subject: [PATCH 03/10] Move vector transfer hoisting tests to a dedicated file
---
.../hoist-redundant-vector-transfers.mlir | 1261 ++++++++++++++++
mlir/test/Dialect/Linalg/hoisting.mlir | 1262 -----------------
2 files changed, 1261 insertions(+), 1262 deletions(-)
create mode 100644 mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
diff --git a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
new file mode 100644
index 0000000000000..ff9889e918c6c
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
@@ -0,0 +1,1261 @@
+// RUN: mlir-opt -transform-interpreter -canonicalize --split-input-file %s | FileCheck %s
+
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested inside a single loop
+// * Indices are constant
+///----------------------------------------------------------------------------------------
+
+// The most basic example - hoisting is safe.
+
+// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index) {
+func.func @hoist_basic_vector_xfer_pair(
+ %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK: %[[VAL_6:.*]] = "test.val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: scf.yield %[[VAL_6]] : vector<1xf32>
+// CHECK: }
+// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is no longer safe. That's due to
+// an extra xfer_write inside the loop.
+
+// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
+ %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: vector.transfer_write %[[IN]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: }
+
+ scf.for %i = %lb to %ub step %step {
+ vector.transfer_write %in, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+
+ %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is no longer safe. That's due to
+// an extra xfer_write into _an alias_ of the %mem Op that is used by the
+// original xfer pair.
+
+// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
+ %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[SV:.*]] = memref.subview %[[MEM]][0, 0] [1, 1] [1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: vector.transfer_write %[[IN]], %[[SV]][%[[C0]], %[[C0]]] {{.*}} : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: }
+
+ %sv = memref.subview %mem[0, 0][1, 1][1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
+ scf.for %i = %lb to %ub step %step {
+ vector.transfer_write %in, %sv[%c0, %c0] : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
+
+ %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Similar as the example above, but the memory access is done via
+// memref.assume_alignment. Hoisting is safe as the only users of the
+// "allignment" Op are the xfer Ops within the loop that we want to hoist.
+
+// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair_with_assume_align(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @hoist_basic_vector_xfer_pair_with_assume_align(
+ %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: }
+// CHECK: vector.transfer_write %[[SCF]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+
+ %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %u0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is not safe due to extra memory
+// access inside the loop via the original memref.
+
+// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
+ %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: "test.mem_use"(%[[MEM]])
+// CHECK: vector.transfer_write %[[READ]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: }
+
+ %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ vector.transfer_write %r0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested in double loops
+// * Indices depend on induction variables
+///----------------------------------------------------------------------------------------
+
+// CHECK-LABEL: func @mem_use_outside
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @mem_use_outside(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: scf.yield %[[USE]] : vector<1xf32>
+// CHECK: }
+// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK: }
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
+ %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
+ }
+ }
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @mem_use_inside_outer_loop
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @mem_use_inside_outer_loop(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: scf.yield %[[USE]] : vector<1xf32>
+// CHECK: }
+// CHECK: vector.transfer_write %[[SCF]], %[[MEM]]{{\[}}%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK: }
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
+ %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
+ }
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested in double loops
+// * Indices are constant
+///----------------------------------------------------------------------------------------
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_write
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_before_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: }
+// CHECK: }
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_after_write
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_after_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK: }
+// CHECK: }
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %r3 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %u3 = "test.val_use"(%r3) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %u3, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_read
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_before_read(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>
+// CHECK: "test.val_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
+// CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>
+// CHECK: }
+// CHECK: }
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
+ %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+ %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+ vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Other tests
+///
+/// TODO: Document
+///----------------------------------------------------------------------------------------
+
+// CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint(
+// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+// CHECK-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
+// CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,
+// CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index,
+// CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index,
+// CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,
+// CHECK-SAME: %[[RANDOM:[a-zA-Z0-9]*]]: index,
+// CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1
+func.func @hoist_vector_transfer_pairs_disjoint(
+ %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>,
+ %memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index,
+ %step: index, %random_index : index, %cmp: i1) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c3 = arith.constant 3 : index
+ %cst = arith.constant 0.0 : f32
+
+// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
+// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
+// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
+// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
+// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
+// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
+// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
+// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
+// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
+// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
+// CHECK: }
+// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
+// CHECK: }
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
+// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32>
+ %r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32>
+ %r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
+ %r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32>
+ %r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
+ %r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
+ %r10 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>
+ %r11 = vector.transfer_read %memref0[%random_index, %random_index], %cst: memref<?x?xf32>, vector<2xf32>
+ %u00 = "test.some_use"(%r00) : (vector<2xf32>) -> vector<2xf32>
+ %u01 = "test.some_use"(%r01) : (vector<2xf32>) -> vector<2xf32>
+ %u20 = "test.some_use"(%r20) : (vector<3xf32>) -> vector<3xf32>
+ %u21 = "test.some_use"(%r21) : (vector<3xf32>) -> vector<3xf32>
+ %u30 = "test.some_use"(%r30) : (vector<4xf32>) -> vector<4xf32>
+ %u31 = "test.some_use"(%r31) : (vector<4xf32>) -> vector<4xf32>
+ %u10 = "test.some_use"(%r10) : (vector<2xf32>) -> vector<2xf32>
+ %u11 = "test.some_use"(%r11) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u00, %memref1[%c0, %c0] : vector<2xf32>, memref<?x?xf32>
+ vector.transfer_write %u01, %memref1[%c0, %c1] : vector<2xf32>, memref<?x?xf32>
+ vector.transfer_write %u20, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>
+ vector.transfer_write %u21, %memref2[%c0, %c3] : vector<3xf32>, memref<?x?xf32>
+ vector.transfer_write %u30, %memref3[%c0, %random_index] : vector<4xf32>, memref<?x?xf32>
+ vector.transfer_write %u31, %memref3[%c1, %random_index] : vector<4xf32>, memref<?x?xf32>
+ vector.transfer_write %u10, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>
+ vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @hoist_vector_transfer_pairs_in_affine_loops(
+// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]+]]: memref<64x64xi32>,
+// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]+]]: memref<64x64xi32>,
+// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]+]]: memref<64x64xi32>) {
+// CHECK: %[[C0:.*]] = arith.constant 0 : i32
+// CHECK: affine.for %[[I:.*]] = 0 to 64 {
+// CHECK: affine.for %[[J:.*]] = 0 to 64 step 16 {
+// CHECK: %[[R0:.*]] = vector.transfer_read %[[MEMREF2]][%[[I]], %[[J]]], %[[C0]] : memref<64x64xi32>, vector<16xi32>
+// CHECK: %[[R:.*]] = affine.for %[[K:.*]] = 0 to 64 iter_args(%[[ACC:.*]] = %[[R0]]) -> (vector<16xi32>) {
+// CHECK: %[[AV:.*]] = vector.transfer_read %[[MEMREF0]][%[[I]], %[[K]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>
+// CHECK: %[[BV:.*]] = vector.transfer_read %[[MEMREF1]][%[[K]], %[[J]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>
+// CHECK: %[[T0:.*]] = arith.muli %[[AV]], %[[BV]] : vector<16xi32>
+// CHECK: %[[T1:.*]] = arith.addi %[[ACC]], %[[T0]] : vector<16xi32>
+// CHECK: affine.yield %[[T1]] : vector<16xi32>
+// CHECK: }
+// CHECK: vector.transfer_write %[[R]], %[[MEMREF2]][%[[I]], %[[J]]] : vector<16xi32>, memref<64x64xi32>
+// CHECK: }
+// CHECK: }
+func.func @hoist_vector_transfer_pairs_in_affine_loops(%memref0: memref<64x64xi32>, %memref1: memref<64x64xi32>, %memref2: memref<64x64xi32>) {
+ %c0_i32 = arith.constant 0 : i32
+ affine.for %arg3 = 0 to 64 {
+ affine.for %arg4 = 0 to 64 step 16 {
+ affine.for %arg5 = 0 to 64 {
+ %0 = vector.transfer_read %memref0[%arg3, %arg5], %c0_i32 {permutation_map = affine_map<(d0, d1) -> (0)>} : memref<64x64xi32>, vector<16xi32>
+ %1 = vector.transfer_read %memref1[%arg5, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
+ %2 = vector.transfer_read %memref2[%arg3, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
+ %3 = arith.muli %0, %1 : vector<16xi32>
+ %4 = arith.addi %2, %3 : vector<16xi32>
+ vector.transfer_write %4, %memref2[%arg3, %arg4] : vector<16xi32>, memref<64x64xi32>
+ }
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @hoist_vector_transfer_read(
+// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG: %[[C128:.+]] = arith.constant 128 : index
+// CHECK-DAG: %[[C1024:.+]] = arith.constant 1024 : index
+// CHECK-DAG: %[[CST:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<32x64xf32>
+// CHECK: %[[ALLOC_0:.+]] = memref.alloc() : memref<32x128xf32>
+// CHECK: %[[CAST:.+]] = memref.cast %[[ALLOC_0]] : memref<32x128xf32> to memref<32x128xf32, strided<[128, 1],
+// CHECK-SAME: offset: ?>>
+// CHECK: %[[D0:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]} :
+// CHECK-SAME: memref<32x64xf32>, vector<32x64xf32>
+// CHECK: scf.for %[[ARG0:.+]] = %[[C0]] to %[[C1024]] step %[[C128]] {
+// CHECK: %[[D1:.+]] = vector.transfer_read %[[ALLOC_0]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]}
+// CHECK-SAME: : memref<32x128xf32>, vector<32x128xf32>
+// CHECK: "test.some_use"(%[[D0]], %[[D1]], %[[CAST]]) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32,
+// CHECK-SAME: strided<[128, 1], offset: ?>>) -> ()
+// CHECK: }
+// CHECK: memref.dealloc %[[ALLOC]] : memref<32x64xf32>
+// CHECK: return
+func.func @hoist_vector_transfer_read() {
+ %c0 = arith.constant 0 : index
+ %c128 = arith.constant 128 : index
+ %c1024 = arith.constant 1024 : index
+ %cst_2 = arith.constant 0.000000e+00 : f32
+ %memref0 = memref.alloc() : memref<32x64xf32>
+ %memref2 = memref.alloc() : memref<32x128xf32>
+ %subview2 = memref.subview %memref2[%c0, %c0] [32, 128] [1, 1]: memref<32x128xf32> to memref<32x128xf32, strided<[128, 1], offset: ?>>
+ scf.for %arg0 = %c0 to %c1024 step %c128 {
+ %2 = vector.transfer_read %memref2[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x128xf32>, vector<32x128xf32>
+ %3 = vector.transfer_read %memref0[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x64xf32>, vector<32x64xf32>
+ "test.some_use"(%3, %2, %subview2) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32, strided<[128, 1], offset: ?>>) -> ()
+ }
+ memref.dealloc %memref0 : memref<32x64xf32>
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// The transfers in this test case cannot be hoisted and replaced by a vector
+// iter_arg because they do not match.
+
+// CHECK-LABEL: func.func @non_matching_transfers(
+// CHECK: scf.for {{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+func.func @non_matching_transfers(%m: memref<6x1x7x32xf32>) {
+ %c0 = arith.constant 0 : index
+ %c1024 = arith.constant 1024 : index
+ %c128 = arith.constant 128 : index
+ %cst = arith.constant dense<5.5> : vector<6x7x32xf32>
+ %cst_0 = arith.constant 0.0 : f32
+ scf.for %iv = %c0 to %c1024 step %c128 {
+ %read = vector.transfer_read %m[%c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>} : memref<6x1x7x32xf32>, vector<6x7x32xf32>
+ %added = arith.addf %read, %cst : vector<6x7x32xf32>
+ %bc = vector.broadcast %added : vector<6x7x32xf32> to vector<1x6x7x32xf32>
+ %tr = vector.transpose %bc, [1, 0, 2, 3] : vector<1x6x7x32xf32> to vector<6x1x7x32xf32>
+ vector.transfer_write %tr, %m[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<6x1x7x32xf32>, memref<6x1x7x32xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @no_hoisting_unknown_bound_loop
+func.func @no_hoisting_unknown_bound_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+
+ // %lb and %ub are unbounded, so do not hoist.
+ // CHECK: scf.for {{.*}} {
+ // CHECK-NEXT: vector.transfer_read
+ // CHECK-NEXT: "test.some_use"
+ scf.for %arg2 = %lb to %ub step %c1 {
+ %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
+ "test.some_use"(%read) : (vector<4xi32>) ->()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop
+func.func @no_hoisting_possibly_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+
+ // %lb_0 is in range [%lb, 8], and %ub_0 is in range [4, %ub].
+ // Since %lb_0 could be greater than %ub_0, do not hoist.
+ %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)
+ %ub_0 = affine.max affine_map<(d0) -> (d0, 4)>(%ub)
+
+ // CHECK: scf.for {{.*}} {
+ // CHECK-NEXT: vector.transfer_read
+ // CHECK-NEXT: "test.some_use"
+ scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
+ %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
+ "test.some_use"(%read) : (vector<4xi32>) ->()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub
+func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub(%memref0: memref<20xi32>, %lb: index, %ub: index) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+
+ // %lb_0 is in range [%lb, 8], and %ub_0 is in range [8, %ub].
+ // Since %lb_0 could be equal to %ub_0, do not hoist.
+ %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)
+ %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)
+
+ // CHECK: scf.for {{.*}} {
+ // CHECK-NEXT: vector.transfer_read
+ // CHECK-NEXT: "test.some_use"
+ scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
+ %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
+ "test.some_use"(%read) : (vector<4xi32>) ->()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @hoisting_non_zero_trip_loop
+func.func @hoisting_non_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+
+ // %lb_0 is in range [%lb, 4], and %ub_0 is in range [8, %ub].
+ // Since %lb_0 is guaranteed to be less than %ub_0, hoisting is possible.
+ %lb_0 = affine.min affine_map<(d0) -> (d0, 4)>(%lb)
+ %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)
+
+ // CHECK: vector.transfer_read
+ // CHECK: scf.for {{.*}} {
+ // CHECK-NEXT: "test.some_use"
+ scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
+ %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
+ "test.some_use"(%read) : (vector<4xi32>) ->()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Regression test - `vector.transfer_read` below should not be hoisted.
+// Indeed, %collapse_shape (written to by `vector.transfer_write`) and %alloca
+// (read by `vector.transfer_read`) alias.
+
+// CHECK-LABEL: func.func @no_hoisting_collapse_shape
+// CHECK: scf.for {{.*}} {
+// CHECK: vector.transfer_write {{.*}} : vector<4xi32>, memref<4xi32>
+// CHECK-NEXT: vector.transfer_read {{.*}} : memref<1x4x1xi32>, vector<1x4x1xi32>
+// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
+// CHECK-NEXT: }
+
+func.func @no_hoisting_collapse_shape(%in_0: memref<1x20x1xi32>, %1: memref<9x1xi32>, %vec: vector<4xi32>) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c4 = arith.constant 4 : index
+ %c20 = arith.constant 20 : index
+ %alloca = memref.alloca() alignment = 64 : memref<1x4x1xi32>
+ scf.for %arg0 = %c0 to %c20 step %c4 {
+ %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
+ %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>
+ vector.transfer_write %vec, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>
+ %read = vector.transfer_read %alloca[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>
+ vector.transfer_write %read, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Regression test - `vector.transfer_read` below should not be hoisted.
+// Indeed, %collapse_shape (read by `vector.transfer_read`) and %alloca
+// (written to by `vector.transfer_write`) alias.
+
+// CHECK-LABEL: func.func @no_hoisting_collapse_shape_2
+// CHECK: scf.for {{.*}} {
+// CHECK: vector.transfer_write
+// CHECK: vector.transfer_read
+
+func.func @no_hoisting_collapse_shape_2(%vec: vector<1x12x1xi32>) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c4 = arith.constant 4 : index
+ %c20 = arith.constant 20 : index
+ %alloca = memref.alloca() alignment = 64 : memref<1x12x1xi32>
+ scf.for %arg0 = %c0 to %c20 step %c4 {
+ %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x12x1xi32> into memref<12xi32>
+ vector.transfer_write %vec, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x12x1xi32>, memref<1x12x1xi32>
+ %read = vector.transfer_read %collapse_shape[%c0], %c0_i32 {in_bounds = [true]} : memref<12xi32>, vector<12xi32>
+ "test.some_use"(%read) : (vector<12xi32>) ->()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Regression test - hoisting the following `vector.transfer_{read|write}` pair
+// would not be safe:
+// %lhs = vector.transfer_read %collapsed_1[%c0]
+// vector.transfer_write %op, %collapsed_1[%c0]
+// That's because the following `vector.transfer_read` reads from the same
+// memory (i.e. `%collapsed_1` and `%collapsed_2` alias):
+// %acc = vector.transfer_read %collapsed_2[%c0]
+
+// CHECK-LABEL: func.func @no_hoisting_write_to_buffer
+// CHECK: scf.for {{.*}} {
+// CHECK: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>
+// CHECK-NEXT: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>
+// CHECK-NEXT: vector.outerproduct {{.*}} : vector<1xi32>, i32
+// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1xi32>, memref<2xi32>
+// CHECK-NEXT: }
+
+func.func @no_hoisting_write_to_buffer(%rhs: i32, %arg1: vector<1xi32>) {
+ %c0_i32 = arith.constant 0 : i32
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
+ %c4 = arith.constant 4 : index
+ %c20 = arith.constant 20 : index
+ %alloca = memref.alloca() alignment = 64 : memref<1x1x2xi32>
+ %cast = memref.cast %alloca : memref<1x1x2xi32> to memref<1x1x2xi32>
+ %collapsed_1 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>
+ scf.for %_ = %c0 to %c20 step %c4 {
+ %collapsed_2 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>
+ %lhs = vector.transfer_read %collapsed_1[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>
+ %acc = vector.transfer_read %collapsed_2[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>
+ %op = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<1xi32>, i32
+ vector.transfer_write %op, %collapsed_1[%c0] {in_bounds = [true]} : vector<1xi32>, memref<2xi32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Test that we can hoist out 1-D read-write pairs whose indices are dynamic values.
+
+// CHECK: #[[$MAP1:.+]] = affine_map<()[s0] -> (s0 + 1)>
+// CHECK: #[[$MAP4:.+]] = affine_map<()[s0] -> (s0 + 4)>
+
+// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
+// CHECK-SAME: (%[[BUFFER:.+]]: memref<?x?xf32>, %{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[I0:.+]]: index)
+
+// CHECK: %[[PLUS1:.+]] = affine.apply #[[$MAP1]]()[%[[I0]]]
+// CHECK: %[[PLUS4:.+]] = affine.apply #[[$MAP4]]()[%[[I0]]]
+// CHECK: %2 = vector.transfer_read %[[BUFFER]][%[[I0]], %[[I0]]]
+// CHECK: %3 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[I0]]]
+// CHECK: %4 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
+// CHECK-COUNT-2: scf.for %{{.+}} = {{.+}} -> (vector<4xf32>, vector<4xf32>, vector<4xf32>)
+// CHECK-COUNT-3: "test.some_use"
+// CHECK-COUNT-2: scf.yield {{.+}} : vector<4xf32>, vector<4xf32>, vector<4xf32>
+// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
+// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[I0]]]
+// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[I0]], %[[I0]]]
+
+func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
+ %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
+ %cst = arith.constant 0.0 : f32
+ %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)
+ %i2 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)
+
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+ // Disjoint leading dim
+ %r1 = vector.transfer_read %buffer[%i1, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+ // Non-overlap trailing dim
+ %r2 = vector.transfer_read %buffer[%i1, %i2], %cst: memref<?x?xf32>, vector<4xf32>
+ %u0 = "test.some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
+ %u1 = "test.some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
+ %u2 = "test.some_use"(%r2) : (vector<4xf32>) -> vector<4xf32>
+ vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
+ vector.transfer_write %u1, %buffer[%i1, %i0] : vector<4xf32>, memref<?x?xf32>
+ vector.transfer_write %u2, %buffer[%i1, %i2] : vector<4xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Test that we cannot hoist out read-write pairs whose indices are overlapping.
+
+// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_overlapping_dynamic
+// CHECK-COUNT-2: scf.for
+// CHECK-COUNT-2: vector.transfer_read
+// CHECK-COUNT-2: vector.transfer_write
+
+func.func @hoist_vector_transfer_pairs_overlapping_dynamic(
+ %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
+ %cst = arith.constant 0.0 : f32
+ %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)
+
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+ // Overlapping range with the above
+ %r1 = vector.transfer_read %buffer[%i0, %i1], %cst: memref<?x?xf32>, vector<4xf32>
+ %u0 = "test.some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
+ %u1 = "test.some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
+ vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
+ vector.transfer_write %u1, %buffer[%i0, %i1] : vector<4xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Test that we can hoist out 2-D read-write pairs whose indices are dynamic values.
+
+// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
+// CHECK-COUNT-3: vector.transfer_read
+// CHECK-COUNT-2: %{{.+}}:3 = scf.for {{.+}} -> (vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>)
+// CHECK-COUNT-2: scf.yield {{.+}} : vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>
+// CHECK-COUNT-3: vector.transfer_write
+// CHECK: return
+
+func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
+ %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index, %i1 : index) {
+ %cst = arith.constant 0.0 : f32
+ %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
+ %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
+ %i4 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 16)>(%i1)
+
+ scf.for %i = %lb to %ub step %step {
+ scf.for %j = %lb to %ub step %step {
+ %r0 = vector.transfer_read %buffer[%i0, %i2], %cst: memref<?x?xf32>, vector<16x8xf32>
+ %r1 = vector.transfer_read %buffer[%i0, %i3], %cst: memref<?x?xf32>, vector<16x8xf32>
+ %r2 = vector.transfer_read %buffer[%i0, %i4], %cst: memref<?x?xf32>, vector<16x8xf32>
+ %u0 = "test.some_use"(%r0) : (vector<16x8xf32>) -> vector<16x8xf32>
+ %u1 = "test.some_use"(%r1) : (vector<16x8xf32>) -> vector<16x8xf32>
+ %u2 = "test.some_use"(%r2) : (vector<16x8xf32>) -> vector<16x8xf32>
+ vector.transfer_write %u2, %buffer[%i0, %i4] : vector<16x8xf32>, memref<?x?xf32>
+ vector.transfer_write %u1, %buffer[%i0, %i3] : vector<16x8xf32>, memref<?x?xf32>
+ vector.transfer_write %u0, %buffer[%i0, %i2] : vector<16x8xf32>, memref<?x?xf32>
+ }
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for hoisting through `memref.subview` views, guarded by a static
+/// footprint disjointness check.
+///----------------------------------------------------------------------------------------
+
+// Disjoint static subviews of one buffer: both transfer pairs are hoisted.
+
+// CHECK-LABEL: func.func @hoist_disjoint_static_subviews(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4xf32>,
+// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0] [2] [1]
+// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][2] [2] [1]
+// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
+// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
+// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2xf32>, vector<2xf32>) {
+// CHECK-NEXT: %[[U0:.*]] = "test.val_use"(%[[I0]])
+// CHECK-NEXT: %[[U1:.*]] = "test.val_use"(%[[I1]])
+// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
+// CHECK-NEXT: }
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
+func.func @hoist_disjoint_static_subviews(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ %sv1 = memref.subview %mem[2][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 2>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 2>>, vector<2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 2>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Overlapping static subviews are not provably disjoint: hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_overlapping_static_subviews(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_overlapping_static_subviews(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ %sv1 = memref.subview %mem[1][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 1>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 1>>, vector<2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 1>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A dynamic (but loop-invariant) subview offset with no other access to the
+// buffer: the pair still hoists, since nothing else can alias it.
+
+// CHECK-LABEL: func.func @hoist_dynamic_offset_subview(
+// CHECK: %[[SV:.*]] = memref.subview
+// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
+// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
+// CHECK: }
+// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
+
+func.func @hoist_dynamic_offset_subview(
+ %mem: memref<4xf32>, %off: index, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv = memref.subview %mem[%off][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: ?>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A subview rooted at a collapse_shape that is the buffer's sole accessor: the
+// pair still hoists, since nothing else can alias it.
+
+// CHECK-LABEL: func.func @hoist_subview_of_collapse_shape(
+// CHECK: %[[COLLAPSE:.*]] = memref.collapse_shape
+// CHECK: %[[SV:.*]] = memref.subview %[[COLLAPSE]]
+// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
+// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
+// CHECK: }
+// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
+
+func.func @hoist_subview_of_collapse_shape(
+ %mem: memref<2x2xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %collapse = memref.collapse_shape %mem [[0, 1]] : memref<2x2xf32> into memref<4xf32>
+ %sv = memref.subview %collapse[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A non-transfer op touching the buffer may alias: hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_subview_extra_memory_effect(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: "test.prevent_hoisting"
+// CHECK: }
+
+func.func @negative_hoist_subview_extra_memory_effect(
+ %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ "test.prevent_hoisting"(%mem) : (memref<4xf32>) -> ()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index eadc4af8ba514..62ebf66761e52 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -1,1070 +1,5 @@
// RUN: mlir-opt -transform-interpreter -canonicalize --split-input-file %s | FileCheck %s
-///----------------------------------------------------------------------------------------
-/// Tests for vector.transfer_read + vector.transfer_write pairs
-///
-/// * Nested inside a single loop
-// * Indices are constant
-///----------------------------------------------------------------------------------------
-
-// The most basic example - hoisting is safe.
-
-// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index) {
-func.func @hoist_basic_vector_xfer_pair(
- %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
-// CHECK: %[[VAL_6:.*]] = "test.val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: scf.yield %[[VAL_6]] : vector<1xf32>
-// CHECK: }
-// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Similar as the example above, but hoisting is no longer safe. That's due to
-// an extra xfer_write inside the loop.
-
-// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
-func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
- %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: vector.transfer_write %[[IN]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: }
-
- scf.for %i = %lb to %ub step %step {
- vector.transfer_write %in, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
-
- %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Similar as the example above, but hoisting is no longer safe. That's due to
-// an extra xfer_write into _an alias_ of the %mem Op that is used by the
-// original xfer pair.
-
-// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
-func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
- %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[SV:.*]] = memref.subview %[[MEM]][0, 0] [1, 1] [1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: vector.transfer_write %[[IN]], %[[SV]][%[[C0]], %[[C0]]] {{.*}} : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: }
-
- %sv = memref.subview %mem[0, 0][1, 1][1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
- scf.for %i = %lb to %ub step %step {
- vector.transfer_write %in, %sv[%c0, %c0] : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
-
- %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Similar as the example above, but the memory access is done via
-// memref.assume_alignment. Hoisting is safe as the only users of the
-// "allignment" Op are the xfer Ops within the loop that we want to hoist.
-
-// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair_with_assume_align(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
-func.func @hoist_basic_vector_xfer_pair_with_assume_align(
- %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: }
-// CHECK: vector.transfer_write %[[SCF]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-
- %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %u0 = "test.val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %u0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Similar as the example above, but hoisting is not safe due to extra memory
-// access inside the loop via the original memref.
-
-// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
-func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
- %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: "test.mem_use"(%[[MEM]])
-// CHECK: vector.transfer_write %[[READ]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: }
-
- %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- vector.transfer_write %r0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-///----------------------------------------------------------------------------------------
-/// Tests for vector.transfer_read + vector.transfer_write pairs
-///
-/// * Nested in double loops
-// * Indices depend on induction variables
-///----------------------------------------------------------------------------------------
-
-// CHECK-LABEL: func @mem_use_outside
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
-func.func @mem_use_outside(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: scf.yield %[[USE]] : vector<1xf32>
-// CHECK: }
-// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
-// CHECK: }
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
- %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
- }
- }
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func @mem_use_inside_outer_loop
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
-func.func @mem_use_inside_outer_loop(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: scf.yield %[[USE]] : vector<1xf32>
-// CHECK: }
-// CHECK: vector.transfer_write %[[SCF]], %[[MEM]]{{\[}}%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
-// CHECK: }
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
- %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
- }
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-///----------------------------------------------------------------------------------------
-/// Tests for vector.transfer_read + vector.transfer_write pairs
-///
-/// * Nested in double loops
-// * Indices are constant
-///----------------------------------------------------------------------------------------
-
-// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_write
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
-func.func @negative_mem_use_inside_inner_loop_before_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
-// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: }
-// CHECK: }
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_after_write
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
-func.func @negative_mem_use_inside_inner_loop_after_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
-// CHECK: %[[USE:.*]] = "test.val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
-// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
-// CHECK: }
-// CHECK: }
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %r3 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %u3 = "test.val_use"(%r3) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %u3, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_read
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)
-func.func @negative_mem_use_inside_inner_loop_before_read(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
-
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
-// CHECK: "test.mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
-// CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>
-// CHECK: "test.val_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
-// CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>
-// CHECK: }
-// CHECK: }
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- "test.mem_use"(%mem) : (memref<?x?xf32>) -> ()
- %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
- %use = "test.val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
- vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-///----------------------------------------------------------------------------------------
-/// Other tests
-///
-/// TODO: Document
-///----------------------------------------------------------------------------------------
-
-// CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint(
-// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,
-// CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index,
-// CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index,
-// CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,
-// CHECK-SAME: %[[RANDOM:[a-zA-Z0-9]*]]: index,
-// CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1
-func.func @hoist_vector_transfer_pairs_disjoint(
- %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>,
- %memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index,
- %step: index, %random_index : index, %cmp: i1) {
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
- %c3 = arith.constant 3 : index
- %cst = arith.constant 0.0 : f32
-
-// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
-// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
-// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
-// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
-// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
-// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
-// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
-// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
-// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
-// CHECK: "test.some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
-// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
-// CHECK: }
-// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
-// CHECK: }
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
-// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32>
- %r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32>
- %r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
- %r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32>
- %r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
- %r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
- %r10 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>
- %r11 = vector.transfer_read %memref0[%random_index, %random_index], %cst: memref<?x?xf32>, vector<2xf32>
- %u00 = "test.some_use"(%r00) : (vector<2xf32>) -> vector<2xf32>
- %u01 = "test.some_use"(%r01) : (vector<2xf32>) -> vector<2xf32>
- %u20 = "test.some_use"(%r20) : (vector<3xf32>) -> vector<3xf32>
- %u21 = "test.some_use"(%r21) : (vector<3xf32>) -> vector<3xf32>
- %u30 = "test.some_use"(%r30) : (vector<4xf32>) -> vector<4xf32>
- %u31 = "test.some_use"(%r31) : (vector<4xf32>) -> vector<4xf32>
- %u10 = "test.some_use"(%r10) : (vector<2xf32>) -> vector<2xf32>
- %u11 = "test.some_use"(%r11) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u00, %memref1[%c0, %c0] : vector<2xf32>, memref<?x?xf32>
- vector.transfer_write %u01, %memref1[%c0, %c1] : vector<2xf32>, memref<?x?xf32>
- vector.transfer_write %u20, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>
- vector.transfer_write %u21, %memref2[%c0, %c3] : vector<3xf32>, memref<?x?xf32>
- vector.transfer_write %u30, %memref3[%c0, %random_index] : vector<4xf32>, memref<?x?xf32>
- vector.transfer_write %u31, %memref3[%c1, %random_index] : vector<4xf32>, memref<?x?xf32>
- vector.transfer_write %u10, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>
- vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func @hoist_vector_transfer_pairs_in_affine_loops(
-// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]+]]: memref<64x64xi32>,
-// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]+]]: memref<64x64xi32>,
-// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]+]]: memref<64x64xi32>) {
-// CHECK: %[[C0:.*]] = arith.constant 0 : i32
-// CHECK: affine.for %[[I:.*]] = 0 to 64 {
-// CHECK: affine.for %[[J:.*]] = 0 to 64 step 16 {
-// CHECK: %[[R0:.*]] = vector.transfer_read %[[MEMREF2]][%[[I]], %[[J]]], %[[C0]] : memref<64x64xi32>, vector<16xi32>
-// CHECK: %[[R:.*]] = affine.for %[[K:.*]] = 0 to 64 iter_args(%[[ACC:.*]] = %[[R0]]) -> (vector<16xi32>) {
-// CHECK: %[[AV:.*]] = vector.transfer_read %[[MEMREF0]][%[[I]], %[[K]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>
-// CHECK: %[[BV:.*]] = vector.transfer_read %[[MEMREF1]][%[[K]], %[[J]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>
-// CHECK: %[[T0:.*]] = arith.muli %[[AV]], %[[BV]] : vector<16xi32>
-// CHECK: %[[T1:.*]] = arith.addi %[[ACC]], %[[T0]] : vector<16xi32>
-// CHECK: affine.yield %[[T1]] : vector<16xi32>
-// CHECK: }
-// CHECK: vector.transfer_write %[[R]], %[[MEMREF2]][%[[I]], %[[J]]] : vector<16xi32>, memref<64x64xi32>
-// CHECK: }
-// CHECK: }
-func.func @hoist_vector_transfer_pairs_in_affine_loops(%memref0: memref<64x64xi32>, %memref1: memref<64x64xi32>, %memref2: memref<64x64xi32>) {
- %c0_i32 = arith.constant 0 : i32
- affine.for %arg3 = 0 to 64 {
- affine.for %arg4 = 0 to 64 step 16 {
- affine.for %arg5 = 0 to 64 {
- %0 = vector.transfer_read %memref0[%arg3, %arg5], %c0_i32 {permutation_map = affine_map<(d0, d1) -> (0)>} : memref<64x64xi32>, vector<16xi32>
- %1 = vector.transfer_read %memref1[%arg5, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
- %2 = vector.transfer_read %memref2[%arg3, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
- %3 = arith.muli %0, %1 : vector<16xi32>
- %4 = arith.addi %2, %3 : vector<16xi32>
- vector.transfer_write %4, %memref2[%arg3, %arg4] : vector<16xi32>, memref<64x64xi32>
- }
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func.func @hoist_vector_transfer_read(
-// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
-// CHECK-DAG: %[[C128:.+]] = arith.constant 128 : index
-// CHECK-DAG: %[[C1024:.+]] = arith.constant 1024 : index
-// CHECK-DAG: %[[CST:.+]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<32x64xf32>
-// CHECK: %[[ALLOC_0:.+]] = memref.alloc() : memref<32x128xf32>
-// CHECK: %[[CAST:.+]] = memref.cast %[[ALLOC_0]] : memref<32x128xf32> to memref<32x128xf32, strided<[128, 1],
-// CHECK-SAME: offset: ?>>
-// CHECK: %[[D0:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]} :
-// CHECK-SAME: memref<32x64xf32>, vector<32x64xf32>
-// CHECK: scf.for %[[ARG0:.+]] = %[[C0]] to %[[C1024]] step %[[C128]] {
-// CHECK: %[[D1:.+]] = vector.transfer_read %[[ALLOC_0]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]}
-// CHECK-SAME: : memref<32x128xf32>, vector<32x128xf32>
-// CHECK: "test.some_use"(%[[D0]], %[[D1]], %[[CAST]]) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32,
-// CHECK-SAME: strided<[128, 1], offset: ?>>) -> ()
-// CHECK: }
-// CHECK: memref.dealloc %[[ALLOC]] : memref<32x64xf32>
-// CHECK: return
-func.func @hoist_vector_transfer_read() {
- %c0 = arith.constant 0 : index
- %c128 = arith.constant 128 : index
- %c1024 = arith.constant 1024 : index
- %cst_2 = arith.constant 0.000000e+00 : f32
- %memref0 = memref.alloc() : memref<32x64xf32>
- %memref2 = memref.alloc() : memref<32x128xf32>
- %subview2 = memref.subview %memref2[%c0, %c0] [32, 128] [1, 1]: memref<32x128xf32> to memref<32x128xf32, strided<[128, 1], offset: ?>>
- scf.for %arg0 = %c0 to %c1024 step %c128 {
- %2 = vector.transfer_read %memref2[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x128xf32>, vector<32x128xf32>
- %3 = vector.transfer_read %memref0[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x64xf32>, vector<32x64xf32>
- "test.some_use"(%3, %2, %subview2) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32, strided<[128, 1], offset: ?>>) -> ()
- }
- memref.dealloc %memref0 : memref<32x64xf32>
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// The transfers in this test case cannot be hoisted and replaced by a vector
-// iter_arg because they do not match.
-
-// CHECK-LABEL: func.func @non_matching_transfers(
-// CHECK: scf.for {{.*}} {
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
-// CHECK: }
-func.func @non_matching_transfers(%m: memref<6x1x7x32xf32>) {
- %c0 = arith.constant 0 : index
- %c1024 = arith.constant 1024 : index
- %c128 = arith.constant 128 : index
- %cst = arith.constant dense<5.5> : vector<6x7x32xf32>
- %cst_0 = arith.constant 0.0 : f32
- scf.for %iv = %c0 to %c1024 step %c128 {
- %read = vector.transfer_read %m[%c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>} : memref<6x1x7x32xf32>, vector<6x7x32xf32>
- %added = arith.addf %read, %cst : vector<6x7x32xf32>
- %bc = vector.broadcast %added : vector<6x7x32xf32> to vector<1x6x7x32xf32>
- %tr = vector.transpose %bc, [1, 0, 2, 3] : vector<1x6x7x32xf32> to vector<6x1x7x32xf32>
- vector.transfer_write %tr, %m[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<6x1x7x32xf32>, memref<6x1x7x32xf32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func.func @no_hoisting_unknown_bound_loop
-func.func @no_hoisting_unknown_bound_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
-
- // %lb and %ub are unbounded, so do not hoist.
- // CHECK: scf.for {{.*}} {
- // CHECK-NEXT: vector.transfer_read
- // CHECK-NEXT: "test.some_use"
- scf.for %arg2 = %lb to %ub step %c1 {
- %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
- "test.some_use"(%read) : (vector<4xi32>) ->()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop
-func.func @no_hoisting_possibly_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
-
- // %lb_0 is in range [%lb, 8], and %ub_0 is in range [4, %ub].
- // Since %lb_0 could be greater than %ub_0, do not hoist.
- %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)
- %ub_0 = affine.max affine_map<(d0) -> (d0, 4)>(%ub)
-
- // CHECK: scf.for {{.*}} {
- // CHECK-NEXT: vector.transfer_read
- // CHECK-NEXT: "test.some_use"
- scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
- %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
- "test.some_use"(%read) : (vector<4xi32>) ->()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub
-func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub(%memref0: memref<20xi32>, %lb: index, %ub: index) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
-
- // %lb_0 is in range [%lb, 8], and %ub_0 is in range [8, %ub].
- // Since %lb_0 could be equal to %ub_0, do not hoist.
- %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)
- %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)
-
- // CHECK: scf.for {{.*}} {
- // CHECK-NEXT: vector.transfer_read
- // CHECK-NEXT: "test.some_use"
- scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
- %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
- "test.some_use"(%read) : (vector<4xi32>) ->()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// CHECK-LABEL: func.func @hoisting_non_zero_trip_loop
-func.func @hoisting_non_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
-
- // %lb_0 is in range [%lb, 4], and %ub_0 is in range [8, %ub].
- // Since %lb_0 is guaranteed to be less than %ub_0, hoisting is possible.
- %lb_0 = affine.min affine_map<(d0) -> (d0, 4)>(%lb)
- %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)
-
- // CHECK: vector.transfer_read
- // CHECK: scf.for {{.*}} {
- // CHECK-NEXT: "test.some_use"
- scf.for %arg2 = %lb_0 to %ub_0 step %c1 {
- %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>
- "test.some_use"(%read) : (vector<4xi32>) ->()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0 verify_non_zero_trip
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Regression test - `vector.transfer_read` below should not be hoisted.
-// Indeed, %collapse_shape (written to by `vector.transfer_write`) and %alloca
-// (read by `vector.transfer_read`) alias.
-
-// CHECK-LABEL: func.func @no_hoisting_collapse_shape
-// CHECK: scf.for {{.*}} {
-// CHECK: vector.transfer_write {{.*}} : vector<4xi32>, memref<4xi32>
-// CHECK-NEXT: vector.transfer_read {{.*}} : memref<1x4x1xi32>, vector<1x4x1xi32>
-// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
-// CHECK-NEXT: }
-
-func.func @no_hoisting_collapse_shape(%in_0: memref<1x20x1xi32>, %1: memref<9x1xi32>, %vec: vector<4xi32>) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c4 = arith.constant 4 : index
- %c20 = arith.constant 20 : index
- %alloca = memref.alloca() alignment = 64 : memref<1x4x1xi32>
- scf.for %arg0 = %c0 to %c20 step %c4 {
- %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
- %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>
- vector.transfer_write %vec, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>
- %read = vector.transfer_read %alloca[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>
- vector.transfer_write %read, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Regression test - `vector.transfer_read` below should not be hoisted.
-// Indeed, %collapse_shape (read by `vector.transfer_read`) and %alloca
-// (written to by `vector.transfer_write`) alias.
-
-// CHECK-LABEL: func.func @no_hoisting_collapse_shape_2
-// CHECK: scf.for {{.*}} {
-// CHECK: vector.transfer_write
-// CHECK: vector.transfer_read
-
-func.func @no_hoisting_collapse_shape_2(%vec: vector<1x12x1xi32>) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c4 = arith.constant 4 : index
- %c20 = arith.constant 20 : index
- %alloca = memref.alloca() alignment = 64 : memref<1x12x1xi32>
- scf.for %arg0 = %c0 to %c20 step %c4 {
- %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x12x1xi32> into memref<12xi32>
- vector.transfer_write %vec, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x12x1xi32>, memref<1x12x1xi32>
- %read = vector.transfer_read %collapse_shape[%c0], %c0_i32 {in_bounds = [true]} : memref<12xi32>, vector<12xi32>
- "test.some_use"(%read) : (vector<12xi32>) ->()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Regression test - hoisting the following `vector.transfer_{read|write}` pair
-// would not be safe:
-// %lhs = vector.transfer_read %collapsed_1[%c0]
-// vector.transfer_write %op, %collapsed_1[%c0]
-// That's because the following `vector.transfer_read` reads from the same
-// memory (i.e. `%collapsed_1` and `%collapsed_2` alias):
-// %acc = vector.transfer_read %collapsed_2[%c0]
-
-// CHECK-LABEL: func.func @no_hoisting_write_to_buffer
-// CHECK: scf.for {{.*}} {
-// CHECK: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>
-// CHECK-NEXT: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>
-// CHECK-NEXT: vector.outerproduct {{.*}} : vector<1xi32>, i32
-// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1xi32>, memref<2xi32>
-// CHECK-NEXT: }
-
-func.func @no_hoisting_write_to_buffer(%rhs: i32, %arg1: vector<1xi32>) {
- %c0_i32 = arith.constant 0 : i32
- %c0 = arith.constant 0 : index
- %c1 = arith.constant 1 : index
- %c4 = arith.constant 4 : index
- %c20 = arith.constant 20 : index
- %alloca = memref.alloca() alignment = 64 : memref<1x1x2xi32>
- %cast = memref.cast %alloca : memref<1x1x2xi32> to memref<1x1x2xi32>
- %collapsed_1 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>
- scf.for %_ = %c0 to %c20 step %c4 {
- %collapsed_2 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>
- %lhs = vector.transfer_read %collapsed_1[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>
- %acc = vector.transfer_read %collapsed_2[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>
- %op = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<1xi32>, i32
- vector.transfer_write %op, %collapsed_1[%c0] {in_bounds = [true]} : vector<1xi32>, memref<2xi32>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Test that we can hoist out 1-D read-write pairs whose indices are dynamic values.
-
-// CHECK: #[[$MAP1:.+]] = affine_map<()[s0] -> (s0 + 1)>
-// CHECK: #[[$MAP4:.+]] = affine_map<()[s0] -> (s0 + 4)>
-
-// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
-// CHECK-SAME: (%[[BUFFER:.+]]: memref<?x?xf32>, %{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[I0:.+]]: index)
-
-// CHECK: %[[PLUS1:.+]] = affine.apply #[[$MAP1]]()[%[[I0]]]
-// CHECK: %[[PLUS4:.+]] = affine.apply #[[$MAP4]]()[%[[I0]]]
-// CHECK: %2 = vector.transfer_read %[[BUFFER]][%[[I0]], %[[I0]]]
-// CHECK: %3 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[I0]]]
-// CHECK: %4 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
-// CHECK-COUNT-2: scf.for %{{.+}} = {{.+}} -> (vector<4xf32>, vector<4xf32>, vector<4xf32>)
-// CHECK-COUNT-3: "test.some_use"
-// CHECK-COUNT-2: scf.yield {{.+}} : vector<4xf32>, vector<4xf32>, vector<4xf32>
-// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
-// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[I0]]]
-// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[I0]], %[[I0]]]
-
-func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
- %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
- %cst = arith.constant 0.0 : f32
- %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)
- %i2 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)
-
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
- // Disjoint leading dim
- %r1 = vector.transfer_read %buffer[%i1, %i0], %cst: memref<?x?xf32>, vector<4xf32>
- // Non-overlap trailing dim
- %r2 = vector.transfer_read %buffer[%i1, %i2], %cst: memref<?x?xf32>, vector<4xf32>
- %u0 = "test.some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
- %u1 = "test.some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
- %u2 = "test.some_use"(%r2) : (vector<4xf32>) -> vector<4xf32>
- vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
- vector.transfer_write %u1, %buffer[%i1, %i0] : vector<4xf32>, memref<?x?xf32>
- vector.transfer_write %u2, %buffer[%i1, %i2] : vector<4xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Test that we cannot hoist out read-write pairs whose indices are overlapping.
-
-// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_overlapping_dynamic
-// CHECK-COUNT-2: scf.for
-// CHECK-COUNT-2: vector.transfer_read
-// CHECK-COUNT-2: vector.transfer_write
-
-func.func @hoist_vector_transfer_pairs_overlapping_dynamic(
- %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
- %cst = arith.constant 0.0 : f32
- %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)
-
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
- // Overlapping range with the above
- %r1 = vector.transfer_read %buffer[%i0, %i1], %cst: memref<?x?xf32>, vector<4xf32>
- %u0 = "test.some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
- %u1 = "test.some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
- vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
- vector.transfer_write %u1, %buffer[%i0, %i1] : vector<4xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Test that we can hoist out 2-D read-write pairs whose indices are dynamic values.
-
-// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
-// CHECK-COUNT-3: vector.transfer_read
-// CHECK-COUNT-2: %{{.+}}:3 = scf.for {{.+}} -> (vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>)
-// CHECK-COUNT-2: scf.yield {{.+}} : vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>
-// CHECK-COUNT-3: vector.transfer_write
-// CHECK: return
-
-func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
- %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index, %i1 : index) {
- %cst = arith.constant 0.0 : f32
- %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
- %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
- %i4 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 16)>(%i1)
-
- scf.for %i = %lb to %ub step %step {
- scf.for %j = %lb to %ub step %step {
- %r0 = vector.transfer_read %buffer[%i0, %i2], %cst: memref<?x?xf32>, vector<16x8xf32>
- %r1 = vector.transfer_read %buffer[%i0, %i3], %cst: memref<?x?xf32>, vector<16x8xf32>
- %r2 = vector.transfer_read %buffer[%i0, %i4], %cst: memref<?x?xf32>, vector<16x8xf32>
- %u0 = "test.some_use"(%r0) : (vector<16x8xf32>) -> vector<16x8xf32>
- %u1 = "test.some_use"(%r1) : (vector<16x8xf32>) -> vector<16x8xf32>
- %u2 = "test.some_use"(%r2) : (vector<16x8xf32>) -> vector<16x8xf32>
- vector.transfer_write %u2, %buffer[%i0, %i4] : vector<16x8xf32>, memref<?x?xf32>
- vector.transfer_write %u1, %buffer[%i0, %i3] : vector<16x8xf32>, memref<?x?xf32>
- vector.transfer_write %u0, %buffer[%i0, %i2] : vector<16x8xf32>, memref<?x?xf32>
- }
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
// Test hoisting of vector.extract/vector.broadcast pairs
// CHECK-LABEL: func.func @hoist_vector_broadcasts
@@ -1171,200 +106,3 @@ module attributes {transform.with_named_sequence} {
transform.yield
}
}
-
-// -----
-
-///----------------------------------------------------------------------------------------
-/// Tests for hoisting through `memref.subview` views, guarded by a static
-/// footprint disjointness check.
-///----------------------------------------------------------------------------------------
-
-// Disjoint static subviews of one buffer: both transfer pairs are hoisted.
-
-// CHECK-LABEL: func.func @hoist_disjoint_static_subviews(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4xf32>,
-// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0] [2] [1]
-// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][2] [2] [1]
-// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
-// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
-// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2xf32>, vector<2xf32>) {
-// CHECK-NEXT: %[[U0:.*]] = "val_use"(%[[I0]])
-// CHECK-NEXT: %[[U1:.*]] = "val_use"(%[[I1]])
-// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
-// CHECK-NEXT: }
-// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
-// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
-func.func @hoist_disjoint_static_subviews(
- %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
- %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- %sv1 = memref.subview %mem[2][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 2>>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
-
- %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 2>>, vector<2xf32>
- %u1 = "val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 2>>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// Overlapping static subviews are not provably disjoint: hoisting is blocked.
-
-// CHECK-LABEL: func.func @negative_hoist_overlapping_static_subviews(
-// CHECK: scf.for {{.*}} step %{{.*}} {
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
-// CHECK: }
-
-func.func @negative_hoist_overlapping_static_subviews(
- %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
- %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- %sv1 = memref.subview %mem[1][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 1>>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
-
- %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 1>>, vector<2xf32>
- %u1 = "val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 1>>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// A dynamic (but loop-invariant) subview offset with no other access to the
-// buffer: the pair still hoists, since nothing else can alias it.
-
-// CHECK-LABEL: func.func @hoist_dynamic_offset_subview(
-// CHECK: %[[SV:.*]] = memref.subview
-// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
-// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
-// CHECK: }
-// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
-
-func.func @hoist_dynamic_offset_subview(
- %mem: memref<4xf32>, %off: index, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
- %sv = memref.subview %mem[%off][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: ?>>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
- %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// A subview rooted at a collapse_shape that is the buffer's sole accessor: the
-// pair still hoists, since nothing else can alias it.
-
-// CHECK-LABEL: func.func @hoist_subview_of_collapse_shape(
-// CHECK: %[[COLLAPSE:.*]] = memref.collapse_shape
-// CHECK: %[[SV:.*]] = memref.subview %[[COLLAPSE]]
-// CHECK: %[[R:.*]] = vector.transfer_read %[[SV]]
-// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[R]]) -> (vector<2xf32>) {
-// CHECK: }
-// CHECK: vector.transfer_write %[[FOR]], %[[SV]]
-
-func.func @hoist_subview_of_collapse_shape(
- %mem: memref<2x2xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
- %collapse = memref.collapse_shape %mem [[0, 1]] : memref<2x2xf32> into memref<4xf32>
- %sv = memref.subview %collapse[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
-
-// -----
-
-// A non-transfer op touching the buffer may alias: hoisting is blocked.
-
-// CHECK-LABEL: func.func @negative_hoist_subview_extra_memory_effect(
-// CHECK: scf.for {{.*}} step %{{.*}} {
-// CHECK: vector.transfer_read
-// CHECK: vector.transfer_write
-// CHECK: "prevent_hoisting"
-// CHECK: }
-
-func.func @negative_hoist_subview_extra_memory_effect(
- %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
- %c0 = arith.constant 0 : index
- %pad = arith.constant 0.0 : f32
- %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
- "prevent_hoisting"(%mem) : (memref<4xf32>) -> ()
- }
- return
-}
-
-module attributes {transform.with_named_sequence} {
- transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
- %0 = transform.structured.match ops{["func.func"]} in %arg1
- : (!transform.any_op) -> !transform.any_op
- transform.structured.hoist_redundant_vector_transfers %0
- : (!transform.any_op) -> !transform.any_op
- transform.yield
- }
-}
>From 87f49b00932e3f065199f67bdfc8384b9a5ceb8b Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Wed, 2 Sep 2026 12:30:11 +0200
Subject: [PATCH 04/10] Change confusing variable names
---
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 28 +++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 6859e4c5e8b7c..2859312fb4e12 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -380,30 +380,30 @@ static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
auto rootTy = dyn_cast<MemRefType>(source.getType());
if (!rootTy || !rootTy.hasStaticShape())
return std::nullopt;
- SubviewFootprint fp;
- fp.root = source;
+ SubviewFootprint footprint;
+ footprint.root = source;
// Seed with the whole-root footprint (offset 0, full extent per dimension).
// Each subview in the chain narrows it below.
- fp.offsets.assign(rootTy.getRank(), 0);
- fp.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
+ footprint.offsets.assign(rootTy.getRank(), 0);
+ footprint.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
// Compose from the outermost subview (closest to the root) inward.
for (SubViewOp sv : llvm::reverse(chain)) {
// Rank-reducing subviews would misalign the per-dimension bookkeeping.
if (sv.getSourceType().getRank() != sv.getType().getRank())
return std::nullopt;
- ArrayRef<int64_t> offs = sv.getStaticOffsets();
- ArrayRef<int64_t> szs = sv.getStaticSizes();
- ArrayRef<int64_t> strides = sv.getStaticStrides();
- if (llvm::any_of(offs, ShapedType::isDynamic) ||
- llvm::any_of(szs, ShapedType::isDynamic) ||
- llvm::any_of(strides, [](int64_t s) { return s != 1; }))
+ ArrayRef<int64_t> staticOffsets = sv.getStaticOffsets();
+ ArrayRef<int64_t> staticSizes = sv.getStaticSizes();
+ ArrayRef<int64_t> staticStrides = sv.getStaticStrides();
+ if (llvm::any_of(staticOffsets, ShapedType::isDynamic) ||
+ llvm::any_of(staticSizes, ShapedType::isDynamic) ||
+ llvm::any_of(staticStrides, [](int64_t s) { return s != 1; }))
return std::nullopt;
- for (unsigned d = 0, e = offs.size(); d < e; ++d) {
- fp.offsets[d] += offs[d];
- fp.sizes[d] = szs[d];
+ for (unsigned d = 0, e = staticOffsets.size(); d < e; ++d) {
+ footprint.offsets[d] += staticOffsets[d];
+ footprint.sizes[d] = staticSizes[d];
}
}
- return fp;
+ return footprint;
}
/// True if two footprints provably cover disjoint memory: they must share the
>From bf2d78a76591489ce9ffcea204a257133a435df4 Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Fri, 4 Sep 2026 12:22:15 +0200
Subject: [PATCH 05/10] Remove useless indirections
---
.../mlir/Dialect/MemRef/Utils/MemRefUtils.h | 24 +++++-----
.../Dialect/Linalg/Transforms/Hoisting.cpp | 12 +----
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 48 +++++++++++--------
3 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index b40919c621b78..4ee4f0ed68b5d 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -198,19 +198,17 @@ LogicalResult resolveSourceIndicesRankReducingSubview(
/// negative.
bool hasNegativeStaticStride(MemRefType memRefTy);
-/// Return "true" when no other access nested in `scope` can alias `base`, so
-/// relocating it is safe with respect to aliasing.
-///
-/// `getAccessedMemref` maps an op to the memref it accesses (null if
-/// unmodelled). Visiting the in-`scope` users of `base`'s underlying buffer, a
-/// user conflicts unless it is effect-free, in `excludedOps`, touches a slice
-/// disjoint from `base`'s static footprint, or only reads when `readsAreSafe`
-/// is true. `excludedOps` are left out of the analysis, typically because the
-/// caller intends to relocate them.
-bool hasNoAliasingAccessInScope(
- Value base, Operation *scope,
- function_ref<Value(Operation *)> getAccessedMemref,
- ArrayRef<Operation *> excludedOps = {}, bool readsAreSafe = false);
+/// Return "true" when no other access nested in `scope` can alias `base`.
+///
+/// The underlying buffer is found by looking through the view ops defining
+/// `base`, and its transitive in-`scope` users are visited. A user conflicts
+/// unless it is effect-free, in `excludedOps`, or each of its memref operands
+/// that resolves to the buffer touches a slice disjoint from `base`'s static
+/// footprint (or, when `readsAreSafe`, is only read). `excludedOps` are left
+/// out of the analysis, typically because the caller intends to relocate them.
+bool hasNoAliasingAccessInScope(Value base, Operation *scope,
+ ArrayRef<Operation *> excludedOps = {},
+ bool readsAreSafe = false);
} // namespace memref
} // namespace mlir
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 18101e51ff6bb..36c3e8007ab64 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -163,13 +163,6 @@ void mlir::linalg::hoistRedundantVectorBroadcasts(RewriterBase &rewriter,
}
}
-/// The memref a vector transfer accesses (a null `Value` for any other op).
-static Value vectorTransferBase(Operation *op) {
- if (auto xfer = dyn_cast<VectorTransferOpInterface>(op))
- return xfer.getBase();
- return {};
-}
-
void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
bool verifyNonZeroTrip) {
bool changed = true;
@@ -264,7 +257,6 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
// Hoisting a lone read is safe as long as no aliasing write remains in
// the loop; other reads never conflict.
if (memref::hasNoAliasingAccessInScope(transferRead.getBase(), loop,
- vectorTransferBase,
/*excludedOps=*/{},
/*readsAreSafe=*/true))
loop.moveOutOfLoop(transferRead);
@@ -298,8 +290,8 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
auto viewAliasingIsSafe = [&]() {
if (!viewAliasingIsSafeCache) {
Operation *hoistedPair[] = {transferRead, transferWrite};
- viewAliasingIsSafeCache = memref::hasNoAliasingAccessInScope(
- base, loop, vectorTransferBase, hoistedPair);
+ viewAliasingIsSafeCache =
+ memref::hasNoAliasingAccessInScope(base, loop, hoistedPair);
}
return *viewAliasingIsSafeCache;
};
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 2859312fb4e12..f743e4aa762f2 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -353,10 +353,16 @@ bool hasNegativeStaticStride(MemRefType memRefTy) {
}
namespace {
-/// Static footprint of a memref value resolved through a chain of
+// TODO: This static footprint modelling and its logic overlap with
+// `SubsetOpInterface`, which is unfortunately tensor-only today; if it gains
+// memref support, look into unifying with it to improve code reuse.
+//
+/// Fully static footprint of a memref value resolved through a chain of
/// rank-preserving, unit-stride `memref.subview` ops: the underlying root
/// memref and, per root dimension, the `[offset, offset + size)` interval it
-/// covers.
+/// covers. Every offset and size should be static; a chain with any dynamic
+/// offset or size (or that otherwise cannot be modelled) has no footprint at
+/// all and should be reported as `std::nullopt`.
struct SubviewFootprint {
Value root;
SmallVector<int64_t> offsets;
@@ -408,8 +414,8 @@ static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
/// True if two footprints provably cover disjoint memory: they must share the
/// same root and be separated along at least one dimension.
-static bool footprintsDisjoint(const SubviewFootprint &a,
- const SubviewFootprint &b) {
+static bool areFootprintDisjoint(const SubviewFootprint &a,
+ const SubviewFootprint &b) {
if (a.root != b.root)
return false;
for (unsigned d = 0, e = a.offsets.size(); d < e; ++d) {
@@ -421,10 +427,9 @@ static bool footprintsDisjoint(const SubviewFootprint &a,
return false;
}
-bool hasNoAliasingAccessInScope(
- Value base, Operation *scope,
- function_ref<Value(Operation *)> getAccessedMemref,
- ArrayRef<Operation *> excludedOps, bool readsAreSafe) {
+bool hasNoAliasingAccessInScope(Value base, Operation *scope,
+ ArrayRef<Operation *> excludedOps,
+ bool readsAreSafe) {
auto baseMemref = dyn_cast<MemrefValue>(base);
if (!baseMemref)
return false;
@@ -450,19 +455,24 @@ bool hasNoAliasingAccessInScope(
continue;
if (!scope->isAncestor(user))
continue;
- // In-`scope`, memory-effecting access to the buffer: it conflicts unless it
- // is a provably-disjoint slice or (when allowed) a read.
- Value accessed = getAccessedMemref(user);
- if (baseFp && accessed) {
- std::optional<SubviewFootprint> userFp =
- resolveSubviewFootprint(accessed);
- if (userFp && footprintsDisjoint(*baseFp, *userFp))
+ // In-`scope`, memory-effecting op: each of its operands that resolves to
+ // the buffer conflicts unless it is a provably-disjoint slice or (when
+ // allowed) is only read.
+ for (Value operand : user->getOperands()) {
+ auto slice = dyn_cast<MemrefValue>(operand);
+ if (!slice || skipViewLikeOps(slice) != buffer)
+ continue;
+ if (baseFp) {
+ std::optional<SubviewFootprint> sliceFp =
+ resolveSubviewFootprint(slice);
+ if (sliceFp && areFootprintDisjoint(*baseFp, *sliceFp))
+ continue;
+ }
+ if (readsAreSafe && isa<MemoryEffectOpInterface>(user) &&
+ !hasEffect<MemoryEffects::Write>(user, slice))
continue;
+ return false;
}
- if (readsAreSafe && accessed &&
- !hasEffect<MemoryEffects::Write>(user, accessed))
- continue;
- return false;
}
return true;
}
>From 345c28dd17409fd07da1c352e310e68826d0476c Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Sat, 5 Sep 2026 15:12:39 +0200
Subject: [PATCH 06/10] Anchor subview footprints at their nearest composable
root
---
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 72 +++++++--------
.../hoist-redundant-vector-transfers.mlir | 92 +++++++++++++++++++
2 files changed, 127 insertions(+), 37 deletions(-)
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index f743e4aa762f2..fafde297ddd78 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -357,12 +357,12 @@ namespace {
// `SubsetOpInterface`, which is unfortunately tensor-only today; if it gains
// memref support, look into unifying with it to improve code reuse.
//
-/// Fully static footprint of a memref value resolved through a chain of
-/// rank-preserving, unit-stride `memref.subview` ops: the underlying root
-/// memref and, per root dimension, the `[offset, offset + size)` interval it
-/// covers. Every offset and size should be static; a chain with any dynamic
-/// offset or size (or that otherwise cannot be modelled) has no footprint at
-/// all and should be reported as `std::nullopt`.
+/// Static footprint of a memref value relative to a `root` memref reached
+/// through a chain of rank-preserving, unit-stride `memref.subview` ops: per
+/// `root` dimension, the `[offset, offset + size)` interval the value covers.
+/// The root is the buffer if the whole chain is static, otherwise the source
+/// of the first subview that cannot be modelled statically. Two footprints are
+/// therefore comparable only when they resolve to the *same* root value.
struct SubviewFootprint {
Value root;
SmallVector<int64_t> offsets;
@@ -370,45 +370,43 @@ struct SubviewFootprint {
};
} // namespace
-/// Resolve `base` through a `memref.subview` chain to its static footprint, or
-/// `std::nullopt` if it cannot be modelled statically.
+/// Resolve `base` through a `memref.subview` chain to a static footprint, or
+/// `std::nullopt` if it cannot be modelled. Composition stops at the first
+/// subview with a dynamic offset/size, a non-unit stride, or a rank reduction;
+/// that subview's own result becomes the footprint root. This keeps two slices
+/// that share a common (possibly dynamically-offset) base comparable through
+/// their static offsets relative to that base.
static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
- SmallVector<SubViewOp> chain;
- Value source = base;
- while (auto sv = source.getDefiningOp<SubViewOp>()) {
- chain.push_back(sv);
- source = sv.getSource();
- }
- // The resolved root must be a genuine buffer, not another kind of view.
- if (isa_and_nonnull<ViewLikeOpInterface>(source.getDefiningOp()))
- return std::nullopt;
-
- auto rootTy = dyn_cast<MemRefType>(source.getType());
- if (!rootTy || !rootTy.hasStaticShape())
+ auto baseTy = dyn_cast<MemRefType>(base.getType());
+ if (!baseTy || llvm::any_of(baseTy.getShape(), ShapedType::isDynamic))
return std::nullopt;
SubviewFootprint footprint;
- footprint.root = source;
- // Seed with the whole-root footprint (offset 0, full extent per dimension).
- // Each subview in the chain narrows it below.
- footprint.offsets.assign(rootTy.getRank(), 0);
- footprint.sizes.assign(rootTy.getShape().begin(), rootTy.getShape().end());
- // Compose from the outermost subview (closest to the root) inward.
- for (SubViewOp sv : llvm::reverse(chain)) {
- // Rank-reducing subviews would misalign the per-dimension bookkeeping.
- if (sv.getSourceType().getRank() != sv.getType().getRank())
- return std::nullopt;
+ // Size is fixed by `base` itself; offset accumulates the static offsets of
+ // the composable subviews walked toward the root.
+ footprint.offsets.assign(baseTy.getRank(), 0);
+ footprint.sizes.assign(baseTy.getShape().begin(), baseTy.getShape().end());
+
+ Value cur = base;
+ while (auto sv = cur.getDefiningOp<SubViewOp>()) {
ArrayRef<int64_t> staticOffsets = sv.getStaticOffsets();
- ArrayRef<int64_t> staticSizes = sv.getStaticSizes();
ArrayRef<int64_t> staticStrides = sv.getStaticStrides();
- if (llvm::any_of(staticOffsets, ShapedType::isDynamic) ||
- llvm::any_of(staticSizes, ShapedType::isDynamic) ||
+ // Rank-reducing or non-static-offset / non-unit-stride subviews cannot be
+ // composed: stop here and use `sv`'s result as the root.
+ if (sv.getSourceType().getRank() != sv.getType().getRank() ||
+ llvm::any_of(staticOffsets, ShapedType::isDynamic) ||
llvm::any_of(staticStrides, [](int64_t s) { return s != 1; }))
- return std::nullopt;
- for (unsigned d = 0, e = staticOffsets.size(); d < e; ++d) {
+ break;
+ for (unsigned d = 0, e = staticOffsets.size(); d < e; ++d)
footprint.offsets[d] += staticOffsets[d];
- footprint.sizes[d] = staticSizes[d];
- }
+ cur = sv.getSource();
}
+ // The root must be a genuine buffer or a `memref.subview` result, not another
+ // kind of view (collapse/expand/reshape) whose coordinate remapping would
+ // invalidate the per-dimension offset bookkeeping.
+ if (auto *def = cur.getDefiningOp())
+ if (isa<ViewLikeOpInterface>(def) && !isa<SubViewOp>(def))
+ return std::nullopt;
+ footprint.root = cur;
return footprint;
}
diff --git a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
index ff9889e918c6c..488d2c93d2b2e 100644
--- a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
+++ b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
@@ -1259,3 +1259,95 @@ module attributes {transform.with_named_sequence} {
transform.yield
}
}
+
+// -----
+
+// Two static slices of the same dynamically-offset subview: they resolve to a
+// shared root and are provably disjoint, so both pairs are hoisted.
+
+// CHECK-LABEL: func.func @hoist_disjoint_subviews_of_dynamic_parent(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<8xf32>,
+// CHECK-SAME: %[[OFF:[a-zA-Z0-9]+]]: index,
+// CHECK: %[[PARENT:.*]] = memref.subview %[[MEM]][%[[OFF]]] [4] [1]
+// CHECK-NEXT: %[[SV0:.*]] = memref.subview %[[PARENT]][0] [2] [1]
+// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[PARENT]][2] [2] [1]
+// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
+// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
+// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2xf32>, vector<2xf32>) {
+// CHECK-NEXT: %[[U0:.*]] = "test.val_use"(%[[I0]])
+// CHECK-NEXT: %[[U1:.*]] = "test.val_use"(%[[I1]])
+// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
+// CHECK-NEXT: }
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
+func.func @hoist_disjoint_subviews_of_dynamic_parent(
+ %mem: memref<8xf32>, %off: index, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %parent = memref.subview %mem[%off][4][1] : memref<8xf32> to memref<4xf32, strided<[1], offset: ?>>
+ %sv0 = memref.subview %parent[0][2][1] : memref<4xf32, strided<[1], offset: ?>> to memref<2xf32, strided<[1], offset: ?>>
+ %sv1 = memref.subview %parent[2][2][1] : memref<4xf32, strided<[1], offset: ?>> to memref<2xf32, strided<[1], offset: ?>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// Slices of two distinct dynamically-offset subviews resolve to distinct roots,
+// so they are not comparable and hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_subviews_of_distinct_dynamic_parents(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_subviews_of_distinct_dynamic_parents(
+ %mem: memref<8xf32>, %off0: index, %off1: index, %lb : index, %ub : index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %parent0 = memref.subview %mem[%off0][4][1] : memref<8xf32> to memref<4xf32, strided<[1], offset: ?>>
+ %parent1 = memref.subview %mem[%off1][4][1] : memref<8xf32> to memref<4xf32, strided<[1], offset: ?>>
+ %sv0 = memref.subview %parent0[0][2][1] : memref<4xf32, strided<[1], offset: ?>> to memref<2xf32, strided<[1], offset: ?>>
+ %sv1 = memref.subview %parent1[0][2][1] : memref<4xf32, strided<[1], offset: ?>> to memref<2xf32, strided<[1], offset: ?>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+
+ %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: ?>>, vector<2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
+ vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: ?>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
>From f62d79f1490668fb59837f26f666343a2e1e9ebc Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Sat, 5 Sep 2026 17:41:23 +0200
Subject: [PATCH 07/10] Prove subview disjointness on static dims despite
dynamic sizes
---
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 35 ++++----
.../hoist-redundant-vector-transfers.mlir | 89 +++++++++++++++++++
2 files changed, 109 insertions(+), 15 deletions(-)
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index fafde297ddd78..372d11488a81d 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -353,16 +353,17 @@ bool hasNegativeStaticStride(MemRefType memRefTy) {
}
namespace {
-// TODO: This static footprint modelling and its logic overlap with
+// TODO: This footprint modelling and its logic overlap with
// `SubsetOpInterface`, which is unfortunately tensor-only today; if it gains
// memref support, look into unifying with it to improve code reuse.
//
-/// Static footprint of a memref value relative to a `root` memref reached
-/// through a chain of rank-preserving, unit-stride `memref.subview` ops: per
-/// `root` dimension, the `[offset, offset + size)` interval the value covers.
-/// The root is the buffer if the whole chain is static, otherwise the source
-/// of the first subview that cannot be modelled statically. Two footprints are
-/// therefore comparable only when they resolve to the *same* root value.
+/// Footprint of a memref value relative to a `root` memref reached through a
+/// chain of rank-preserving, unit-stride `memref.subview` ops: per `root`
+/// dimension, the `[offset, offset + size)` interval the value covers (a `size`
+/// may be dynamic). The root is the underlying buffer if the whole chain
+/// composes, otherwise the result of the first subview (walking toward the
+/// root) that cannot be composed. Two footprints are therefore comparable only
+/// when they resolve to the *same* root value.
struct SubviewFootprint {
Value root;
SmallVector<int64_t> offsets;
@@ -370,15 +371,16 @@ struct SubviewFootprint {
};
} // namespace
-/// Resolve `base` through a `memref.subview` chain to a static footprint, or
+/// Resolve `base` through a `memref.subview` chain to a footprint, or
/// `std::nullopt` if it cannot be modelled. Composition stops at the first
-/// subview with a dynamic offset/size, a non-unit stride, or a rank reduction;
-/// that subview's own result becomes the footprint root. This keeps two slices
-/// that share a common (possibly dynamically-offset) base comparable through
-/// their static offsets relative to that base.
+/// subview with a dynamic offset, a non-unit stride, or a rank reduction; that
+/// subview's own result becomes the footprint root. This keeps two slices that
+/// share a common (possibly dynamically-offset) base comparable through their
+/// static offsets relative to that base. Sizes may be dynamic:
+/// such dimensions simply cannot be used to prove disjointness.
static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
auto baseTy = dyn_cast<MemRefType>(base.getType());
- if (!baseTy || llvm::any_of(baseTy.getShape(), ShapedType::isDynamic))
+ if (!baseTy)
return std::nullopt;
SubviewFootprint footprint;
// Size is fixed by `base` itself; offset accumulates the static offsets of
@@ -411,12 +413,15 @@ static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
}
/// True if two footprints provably cover disjoint memory: they must share the
-/// same root and be separated along at least one dimension.
+/// same root and be separated along at least one statically-sized dimension.
static bool areFootprintDisjoint(const SubviewFootprint &a,
const SubviewFootprint &b) {
if (a.root != b.root)
return false;
for (unsigned d = 0, e = a.offsets.size(); d < e; ++d) {
+ // A dimension can prove disjointness only when both extents are static.
+ if (ShapedType::isDynamic(a.sizes[d]) || ShapedType::isDynamic(b.sizes[d]))
+ continue;
int64_t aHi = a.offsets[d] + a.sizes[d];
int64_t bHi = b.offsets[d] + b.sizes[d];
if (aHi <= b.offsets[d] || bHi <= a.offsets[d])
@@ -432,7 +437,7 @@ bool hasNoAliasingAccessInScope(Value base, Operation *scope,
if (!baseMemref)
return false;
Value buffer = skipViewLikeOps(baseMemref);
- // Static footprint of `base`, if modellable; enables the disjointness escape.
+ // Footprint of `base`, if modellable; enables the disjointness escape.
std::optional<SubviewFootprint> baseFp = resolveSubviewFootprint(base);
// Visit the transitive users of the buffer, following views.
diff --git a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
index 488d2c93d2b2e..3f04ea125e404 100644
--- a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
+++ b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
@@ -1351,3 +1351,92 @@ module attributes {transform.with_named_sequence} {
transform.yield
}
}
+
+// -----
+
+// A subview with a dynamic size: disjointness is proven along the static
+// dimension (rows [0, 2) vs [2, 4)).
+
+// CHECK-LABEL: func.func @hoist_disjoint_dynamic_size_subviews(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4x8xf32>,
+// CHECK-SAME: %[[SZ:[a-zA-Z0-9]+]]: index,
+// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0, 0] [2, %[[SZ]]] [1, 1]
+// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][2, 0] [2, %[[SZ]]] [1, 1]
+// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
+// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
+// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2x4xf32>, vector<2x4xf32>) {
+// CHECK-NEXT: %[[U0:.*]] = "test.val_use"(%[[I0]])
+// CHECK-NEXT: %[[U1:.*]] = "test.val_use"(%[[I1]])
+// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
+// CHECK-NEXT: }
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
+// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
+func.func @hoist_disjoint_dynamic_size_subviews(
+ %mem: memref<4x8xf32>, %sz: index, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0, 0][2, %sz][1, 1] : memref<4x8xf32> to memref<2x?xf32, strided<[8, 1]>>
+ %sv1 = memref.subview %mem[2, 0][2, %sz][1, 1] : memref<4x8xf32> to memref<2x?xf32, strided<[8, 1], offset: 16>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0, %c0], %pad : memref<2x?xf32, strided<[8, 1]>>, vector<2x4xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2x4xf32>) -> vector<2x4xf32>
+ vector.transfer_write %u0, %sv0[%c0, %c0] : vector<2x4xf32>, memref<2x?xf32, strided<[8, 1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0, %c0], %pad : memref<2x?xf32, strided<[8, 1], offset: 16>>, vector<2x4xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2x4xf32>) -> vector<2x4xf32>
+ vector.transfer_write %u1, %sv1[%c0, %c0] : vector<2x4xf32>, memref<2x?xf32, strided<[8, 1], offset: 16>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// The only separation is along the dynamic-size dimension (offsets 0 vs 4),
+// while the static dimension is identical. The dynamic dimension cannot prove
+// disjointness, so hoisting is blocked.
+
+// CHECK-LABEL: func.func @negative_hoist_dynamic_size_only_separation(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: vector.transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+
+func.func @negative_hoist_dynamic_size_only_separation(
+ %mem: memref<4x8xf32>, %sz: index, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ %sv0 = memref.subview %mem[0, 0][2, %sz][1, 1] : memref<4x8xf32> to memref<2x?xf32, strided<[8, 1]>>
+ %sv1 = memref.subview %mem[0, 4][2, %sz][1, 1] : memref<4x8xf32> to memref<2x?xf32, strided<[8, 1], offset: 4>>
+ scf.for %i = %lb to %ub step %step {
+ %r0 = vector.transfer_read %sv0[%c0, %c0], %pad : memref<2x?xf32, strided<[8, 1]>>, vector<2x4xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2x4xf32>) -> vector<2x4xf32>
+ vector.transfer_write %u0, %sv0[%c0, %c0] : vector<2x4xf32>, memref<2x?xf32, strided<[8, 1]>>
+
+ %r1 = vector.transfer_read %sv1[%c0, %c0], %pad : memref<2x?xf32, strided<[8, 1], offset: 4>>, vector<2x4xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2x4xf32>) -> vector<2x4xf32>
+ vector.transfer_write %u1, %sv1[%c0, %c0] : vector<2x4xf32>, memref<2x?xf32, strided<[8, 1], offset: 4>>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
>From b9fbfb96b538cdd8d03c5aecaf440f0f9ba69fbd Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Wed, 9 Sep 2026 10:01:30 +0200
Subject: [PATCH 08/10] Name subview footprint root buffer, clarify wording
---
mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 34 ++++++++++---------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 372d11488a81d..dd39a95327b7a 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -358,9 +358,10 @@ namespace {
// memref support, look into unifying with it to improve code reuse.
//
/// Footprint of a memref value relative to a `root` memref reached through a
-/// chain of rank-preserving, unit-stride `memref.subview` ops: per `root`
-/// dimension, the `[offset, offset + size)` interval the value covers (a `size`
-/// may be dynamic). The root is the underlying buffer if the whole chain
+/// chain of rank-preserving `memref.subview` ops whose `strides` operand is all
+/// ones (so each result index steps one source element in every dimension): per
+/// `root` dimension, the `[offset, offset + size)` interval the value covers (a
+/// `size` may be dynamic). The root is the underlying buffer if the whole chain
/// composes, otherwise the result of the first subview (walking toward the
/// root) that cannot be composed. Two footprints are therefore comparable only
/// when they resolve to the *same* root value.
@@ -373,11 +374,11 @@ struct SubviewFootprint {
/// Resolve `base` through a `memref.subview` chain to a footprint, or
/// `std::nullopt` if it cannot be modelled. Composition stops at the first
-/// subview with a dynamic offset, a non-unit stride, or a rank reduction; that
-/// subview's own result becomes the footprint root. This keeps two slices that
-/// share a common (possibly dynamically-offset) base comparable through their
-/// static offsets relative to that base. Sizes may be dynamic:
-/// such dimensions simply cannot be used to prove disjointness.
+/// subview with a dynamic offset, a `strides` operand other than all ones, or a
+/// rank reduction; that subview's own result becomes the footprint root. This
+/// keeps two slices that share a common (possibly dynamically-offset) base
+/// comparable through their static offsets relative to that base. Sizes may be
+/// dynamic: such dimensions simply cannot be used to prove disjointness.
static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
auto baseTy = dyn_cast<MemRefType>(base.getType());
if (!baseTy)
@@ -392,8 +393,9 @@ static std::optional<SubviewFootprint> resolveSubviewFootprint(Value base) {
while (auto sv = cur.getDefiningOp<SubViewOp>()) {
ArrayRef<int64_t> staticOffsets = sv.getStaticOffsets();
ArrayRef<int64_t> staticStrides = sv.getStaticStrides();
- // Rank-reducing or non-static-offset / non-unit-stride subviews cannot be
- // composed: stop here and use `sv`'s result as the root.
+ // Rank-reducing subviews, or those with a dynamic offset or a `strides`
+ // operand other than all ones, cannot be composed: stop here and use `sv`'s
+ // result as the root.
if (sv.getSourceType().getRank() != sv.getType().getRank() ||
llvm::any_of(staticOffsets, ShapedType::isDynamic) ||
llvm::any_of(staticStrides, [](int64_t s) { return s != 1; }))
@@ -436,13 +438,13 @@ bool hasNoAliasingAccessInScope(Value base, Operation *scope,
auto baseMemref = dyn_cast<MemrefValue>(base);
if (!baseMemref)
return false;
- Value buffer = skipViewLikeOps(baseMemref);
+ Value rootBuffer = skipViewLikeOps(baseMemref);
// Footprint of `base`, if modellable; enables the disjointness escape.
std::optional<SubviewFootprint> baseFp = resolveSubviewFootprint(base);
- // Visit the transitive users of the buffer, following views.
- SmallVector<Operation *> worklist(buffer.getUsers().begin(),
- buffer.getUsers().end());
+ // Visit the transitive users of the root buffer, following views.
+ SmallVector<Operation *> worklist(rootBuffer.getUsers().begin(),
+ rootBuffer.getUsers().end());
SmallPtrSet<Operation *, 8> processed;
while (!worklist.empty()) {
Operation *user = worklist.pop_back_val();
@@ -459,11 +461,11 @@ bool hasNoAliasingAccessInScope(Value base, Operation *scope,
if (!scope->isAncestor(user))
continue;
// In-`scope`, memory-effecting op: each of its operands that resolves to
- // the buffer conflicts unless it is a provably-disjoint slice or (when
+ // the root buffer conflicts unless it is a provably-disjoint slice or (when
// allowed) is only read.
for (Value operand : user->getOperands()) {
auto slice = dyn_cast<MemrefValue>(operand);
- if (!slice || skipViewLikeOps(slice) != buffer)
+ if (!slice || skipViewLikeOps(slice) != rootBuffer)
continue;
if (baseFp) {
std::optional<SubviewFootprint> sliceFp =
>From c169cf85d5ccb5364f8e722d964661e255074374 Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Wed, 9 Sep 2026 10:06:04 +0200
Subject: [PATCH 09/10] Cover rank-3 subview disjointness in hoisting tests
---
.../hoist-redundant-vector-transfers.mlir | 79 ++++++++++---------
1 file changed, 41 insertions(+), 38 deletions(-)
diff --git a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
index 3f04ea125e404..ced8840d5c892 100644
--- a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
+++ b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
@@ -1067,38 +1067,40 @@ module attributes {transform.with_named_sequence} {
///----------------------------------------------------------------------------------------
/// Tests for hoisting through `memref.subview` views, guarded by a static
-/// footprint disjointness check.
+/// footprint disjointness check. Function names carry a `_2d` / `_3d` suffix
+/// marking the rank of the sliced buffer.
///----------------------------------------------------------------------------------------
-// Disjoint static subviews of one buffer: both transfer pairs are hoisted.
+// Disjoint static subviews of one buffer, separated along the middle (a
+// non-trailing) dimension of a rank-3 buffer: both transfer pairs are hoisted.
-// CHECK-LABEL: func.func @hoist_disjoint_static_subviews(
-// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4xf32>,
-// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0] [2] [1]
-// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][2] [2] [1]
+// CHECK-LABEL: func.func @hoist_disjoint_static_subviews_3d(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<2x4x2xf32>,
+// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0, 0, 0] [2, 2, 2] [1, 1, 1]
+// CHECK-NEXT: %[[SV1:.*]] = memref.subview %[[MEM]][0, 2, 0] [2, 2, 2] [1, 1, 1]
// CHECK-NEXT: %[[R0:.*]] = vector.transfer_read %[[SV0]]
// CHECK-NEXT: %[[R1:.*]] = vector.transfer_read %[[SV1]]
-// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2xf32>, vector<2xf32>) {
+// CHECK-NEXT: %[[FOR:.*]]:2 = scf.for {{.*}} iter_args(%[[I0:.*]] = %[[R0]], %[[I1:.*]] = %[[R1]]) -> (vector<2x2x2xf32>, vector<2x2x2xf32>) {
// CHECK-NEXT: %[[U0:.*]] = "test.val_use"(%[[I0]])
// CHECK-NEXT: %[[U1:.*]] = "test.val_use"(%[[I1]])
// CHECK-NEXT: scf.yield %[[U0]], %[[U1]]
// CHECK-NEXT: }
// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
-func.func @hoist_disjoint_static_subviews(
- %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+func.func @hoist_disjoint_static_subviews_3d(
+ %mem: memref<2x4x2xf32>, %lb : index, %ub : index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
- %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- %sv1 = memref.subview %mem[2][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 2>>
+ %sv0 = memref.subview %mem[0, 0, 0][2, 2, 2][1, 1, 1] : memref<2x4x2xf32> to memref<2x2x2xf32, strided<[8, 2, 1]>>
+ %sv1 = memref.subview %mem[0, 2, 0][2, 2, 2][1, 1, 1] : memref<2x4x2xf32> to memref<2x2x2xf32, strided<[8, 2, 1], offset: 4>>
scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ %r0 = vector.transfer_read %sv0[%c0, %c0, %c0], %pad : memref<2x2x2xf32, strided<[8, 2, 1]>>, vector<2x2x2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2x2x2xf32>) -> vector<2x2x2xf32>
+ vector.transfer_write %u0, %sv0[%c0, %c0, %c0] : vector<2x2x2xf32>, memref<2x2x2xf32, strided<[8, 2, 1]>>
- %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 2>>, vector<2xf32>
- %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 2>>
+ %r1 = vector.transfer_read %sv1[%c0, %c0, %c0], %pad : memref<2x2x2xf32, strided<[8, 2, 1], offset: 4>>, vector<2x2x2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2x2x2xf32>) -> vector<2x2x2xf32>
+ vector.transfer_write %u1, %sv1[%c0, %c0, %c0] : vector<2x2x2xf32>, memref<2x2x2xf32, strided<[8, 2, 1], offset: 4>>
}
return
}
@@ -1115,9 +1117,10 @@ module attributes {transform.with_named_sequence} {
// -----
-// Overlapping static subviews are not provably disjoint: hoisting is blocked.
+// Overlapping static subviews, sharing rows along the middle dimension of a
+// rank-3 buffer, are not provably disjoint: hoisting is blocked.
-// CHECK-LABEL: func.func @negative_hoist_overlapping_static_subviews(
+// CHECK-LABEL: func.func @negative_hoist_overlapping_static_subviews_3d(
// CHECK: scf.for {{.*}} step %{{.*}} {
// CHECK: vector.transfer_read
// CHECK: vector.transfer_write
@@ -1125,20 +1128,20 @@ module attributes {transform.with_named_sequence} {
// CHECK: vector.transfer_write
// CHECK: }
-func.func @negative_hoist_overlapping_static_subviews(
- %mem: memref<4xf32>, %lb : index, %ub : index, %step: index) {
+func.func @negative_hoist_overlapping_static_subviews_3d(
+ %mem: memref<2x4x2xf32>, %lb : index, %ub : index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
- %sv0 = memref.subview %mem[0][2][1] : memref<4xf32> to memref<2xf32, strided<[1]>>
- %sv1 = memref.subview %mem[1][2][1] : memref<4xf32> to memref<2xf32, strided<[1], offset: 1>>
+ %sv0 = memref.subview %mem[0, 0, 0][2, 2, 2][1, 1, 1] : memref<2x4x2xf32> to memref<2x2x2xf32, strided<[8, 2, 1]>>
+ %sv1 = memref.subview %mem[0, 1, 0][2, 2, 2][1, 1, 1] : memref<2x4x2xf32> to memref<2x2x2xf32, strided<[8, 2, 1], offset: 2>>
scf.for %i = %lb to %ub step %step {
- %r0 = vector.transfer_read %sv0[%c0], %pad : memref<2xf32, strided<[1]>>, vector<2xf32>
- %u0 = "test.val_use"(%r0) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u0, %sv0[%c0] : vector<2xf32>, memref<2xf32, strided<[1]>>
+ %r0 = vector.transfer_read %sv0[%c0, %c0, %c0], %pad : memref<2x2x2xf32, strided<[8, 2, 1]>>, vector<2x2x2xf32>
+ %u0 = "test.val_use"(%r0) : (vector<2x2x2xf32>) -> vector<2x2x2xf32>
+ vector.transfer_write %u0, %sv0[%c0, %c0, %c0] : vector<2x2x2xf32>, memref<2x2x2xf32, strided<[8, 2, 1]>>
- %r1 = vector.transfer_read %sv1[%c0], %pad : memref<2xf32, strided<[1], offset: 1>>, vector<2xf32>
- %u1 = "test.val_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
- vector.transfer_write %u1, %sv1[%c0] : vector<2xf32>, memref<2xf32, strided<[1], offset: 1>>
+ %r1 = vector.transfer_read %sv1[%c0, %c0, %c0], %pad : memref<2x2x2xf32, strided<[8, 2, 1], offset: 2>>, vector<2x2x2xf32>
+ %u1 = "test.val_use"(%r1) : (vector<2x2x2xf32>) -> vector<2x2x2xf32>
+ vector.transfer_write %u1, %sv1[%c0, %c0, %c0] : vector<2x2x2xf32>, memref<2x2x2xf32, strided<[8, 2, 1], offset: 2>>
}
return
}
@@ -1354,10 +1357,10 @@ module attributes {transform.with_named_sequence} {
// -----
-// A subview with a dynamic size: disjointness is proven along the static
-// dimension (rows [0, 2) vs [2, 4)).
+// A subview with a dynamic size in a rank-2 buffer: disjointness is proven along
+// the static dimension (rows [0, 2) vs [2, 4)).
-// CHECK-LABEL: func.func @hoist_disjoint_dynamic_size_subviews(
+// CHECK-LABEL: func.func @hoist_disjoint_dynamic_size_subviews_2d(
// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<4x8xf32>,
// CHECK-SAME: %[[SZ:[a-zA-Z0-9]+]]: index,
// CHECK: %[[SV0:.*]] = memref.subview %[[MEM]][0, 0] [2, %[[SZ]]] [1, 1]
@@ -1371,7 +1374,7 @@ module attributes {transform.with_named_sequence} {
// CHECK-NEXT: }
// CHECK-NEXT: vector.transfer_write %[[FOR]]#1, %[[SV1]]
// CHECK-NEXT: vector.transfer_write %[[FOR]]#0, %[[SV0]]
-func.func @hoist_disjoint_dynamic_size_subviews(
+func.func @hoist_disjoint_dynamic_size_subviews_2d(
%mem: memref<4x8xf32>, %sz: index, %lb: index, %ub: index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
@@ -1401,11 +1404,11 @@ module attributes {transform.with_named_sequence} {
// -----
-// The only separation is along the dynamic-size dimension (offsets 0 vs 4),
-// while the static dimension is identical. The dynamic dimension cannot prove
-// disjointness, so hoisting is blocked.
+// In a rank-2 buffer, the only separation is along the dynamic-size dimension
+// (offsets 0 vs 4), while the static dimension is identical. The dynamic
+// dimension cannot prove disjointness, so hoisting is blocked.
-// CHECK-LABEL: func.func @negative_hoist_dynamic_size_only_separation(
+// CHECK-LABEL: func.func @negative_hoist_dynamic_size_only_separation_2d(
// CHECK: scf.for {{.*}} step %{{.*}} {
// CHECK: vector.transfer_read
// CHECK: vector.transfer_write
@@ -1413,7 +1416,7 @@ module attributes {transform.with_named_sequence} {
// CHECK: vector.transfer_write
// CHECK: }
-func.func @negative_hoist_dynamic_size_only_separation(
+func.func @negative_hoist_dynamic_size_only_separation_2d(
%mem: memref<4x8xf32>, %sz: index, %lb: index, %ub: index, %step: index) {
%c0 = arith.constant 0 : index
%pad = arith.constant 0.0 : f32
>From 17655d675ccce5ce50e1fdd5d1da44201db51b2e Mon Sep 17 00:00:00 2001
From: Hugo Pompougnac <tech at hugo-pompougnac.fr>
Date: Sat, 5 Sep 2026 20:37:24 +0200
Subject: [PATCH 10/10] [mlir][linalg] Hoist masked vector transfer pairs
---
.../Dialect/Linalg/Transforms/Hoisting.cpp | 69 +++++-
.../hoist-redundant-vector-transfers.mlir | 226 ++++++++++++++++++
2 files changed, 282 insertions(+), 13 deletions(-)
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 36c3e8007ab64..81fe09eca1c16 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -217,9 +217,14 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
LLVM_DEBUG(DBGS() << "Candidate for hoisting: "
<< *transferRead.getOperation() << "\n");
- auto loop = dyn_cast<LoopLikeOpInterface>(transferRead->getParentOp());
- LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead->getParentOp()
- << "\n");
+ // A masked transfer is hoisted as its enclosing `vector.mask` op; that op
+ // is the unit whose parent must be the loop.
+ Operation *readUnit = transferRead;
+ if (auto mask = dyn_cast<vector::MaskOp>(transferRead->getParentOp()))
+ if (mask.getMaskableOp() == transferRead.getOperation())
+ readUnit = mask.getOperation();
+ auto loop = dyn_cast<LoopLikeOpInterface>(readUnit->getParentOp());
+ LLVM_DEBUG(DBGS() << "Parent op: " << *readUnit->getParentOp() << "\n");
if (!isa_and_nonnull<scf::ForOp, affine::AffineForOp>(loop))
return WalkResult::advance();
@@ -233,13 +238,25 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
<< "\n");
SetVector<Operation *> forwardSlice;
- getForwardSlice(transferRead.getOperation(), &forwardSlice);
+ getForwardSlice(readUnit, &forwardSlice);
+ // A forward slice does not follow a value out of a `vector.mask` region,
+ // so re-slice from each enclosing mask op to reach the write.
+ for (unsigned i = 0; i < forwardSlice.size(); ++i) {
+ auto mask = forwardSlice[i]->getParentOfType<vector::MaskOp>();
+ if (mask && forwardSlice.insert(mask.getOperation()))
+ getForwardSlice(mask.getOperation(), &forwardSlice);
+ }
// Look for the last TransferWriteOp in the forwardSlice of
- // `transferRead` that operates on the same memref.
+ // `transferRead` that operates on the same memref. A masked write appears
+ // as the `vector.mask` op wrapping it.
vector::TransferWriteOp transferWrite;
for (auto *sliceOp : llvm::reverse(forwardSlice)) {
auto candidateWrite = dyn_cast<vector::TransferWriteOp>(sliceOp);
+ if (!candidateWrite)
+ if (auto mask = dyn_cast<vector::MaskOp>(sliceOp))
+ candidateWrite =
+ dyn_cast_or_null<vector::TransferWriteOp>(mask.getMaskableOp());
if (!candidateWrite ||
candidateWrite.getBase() != transferRead.getBase())
continue;
@@ -250,6 +267,16 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
for (auto operand : transferRead.getOperands())
if (!loop.isDefinedOutsideOfLoop(operand))
return WalkResult::advance();
+ // A masked read is hoisted as its `vector.mask` op; its mask and passthru
+ // must be loop-invariant too.
+ auto readMask = dyn_cast<vector::MaskOp>(readUnit);
+ if (readMask) {
+ if (!loop.isDefinedOutsideOfLoop(readMask.getMask()))
+ return WalkResult::advance();
+ if (readMask.getPassthru() &&
+ !loop.isDefinedOutsideOfLoop(readMask.getPassthru()))
+ return WalkResult::advance();
+ }
// Only hoist transfer_read / transfer_write pairs and singleton
// transfer_reads for now.
@@ -259,10 +286,23 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
if (memref::hasNoAliasingAccessInScope(transferRead.getBase(), loop,
/*excludedOps=*/{},
/*readsAreSafe=*/true))
- loop.moveOutOfLoop(transferRead);
+ loop.moveOutOfLoop(readUnit);
return WalkResult::advance();
}
+ // The write is hoisted with its own `vector.mask` op, if any. A masked
+ // read must pair with a masked write carrying the same mask (and an
+ // unmasked read with an unmasked write).
+ Operation *writeUnit = transferWrite;
+ if (auto mask = dyn_cast<vector::MaskOp>(transferWrite->getParentOp()))
+ if (mask.getMaskableOp() == transferWrite.getOperation())
+ writeUnit = mask.getOperation();
+ auto writeMask = dyn_cast<vector::MaskOp>(writeUnit);
+ if (!!readMask != !!writeMask)
+ return WalkResult::advance();
+ if (readMask && readMask.getMask() != writeMask.getMask())
+ return WalkResult::advance();
+
LLVM_DEBUG(DBGS() << "Candidate: " << *transferWrite.getOperation()
<< "\n");
@@ -330,8 +370,10 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
// Check 3.
// TODO: may want to memoize this information for performance but it
// likely gets invalidated often.
+ // Compare the hoisted units (the `vector.mask` ops when masked) since the
+ // inner transfers live in sibling mask regions and never dominate.
DominanceInfo dom(loop);
- if (!dom.properlyDominates(transferRead.getOperation(), transferWrite))
+ if (!dom.properlyDominates(readUnit, writeUnit))
return WalkResult::advance();
for (auto &use : transferRead.getBase().getUses()) {
if (!loop->isAncestor(use.getOwner()))
@@ -360,14 +402,15 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
}
}
- // Hoist read before.
- loop.moveOutOfLoop(transferRead);
+ // Hoist read before (with its mask op, if masked).
+ loop.moveOutOfLoop(readUnit);
- // Hoist write after.
- transferWrite->moveAfter(loop);
+ // Hoist write after (with its mask op, if masked).
+ writeUnit->moveAfter(loop);
// Rewrite `loop` with new yields by cloning and erase the original
- // loop.
+ // loop. The carried value is the read unit's result (the masked read
+ // result when masked) and the yielded value is what the write stores.
IRRewriter rewriter(transferRead.getContext());
NewYieldValuesFn yieldFn = [&](OpBuilder &b, Location loc,
ArrayRef<BlockArgument> newBBArgs) {
@@ -375,7 +418,7 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
};
auto maybeNewLoop = loop.replaceWithAdditionalYields(
- rewriter, transferRead.getVector(),
+ rewriter, readUnit->getResult(0),
/*replaceInitOperandUsesInLoop=*/true, yieldFn);
if (failed(maybeNewLoop))
return WalkResult::interrupt();
diff --git a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
index ced8840d5c892..75b6c4629c0c5 100644
--- a/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
+++ b/mlir/test/Dialect/Linalg/hoist-redundant-vector-transfers.mlir
@@ -1443,3 +1443,229 @@ module attributes {transform.with_named_sequence} {
transform.yield
}
}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for hoisting masked (`vector.mask`-wrapped) transfer pairs
+///----------------------------------------------------------------------------------------
+
+// A masked read/write pair sharing the same mask is hoisted as a whole: the
+// masked read/write move to the loop boundary while the value is carried
+// unmasked through an iter_arg.
+
+// CHECK-LABEL: func.func @hoist_masked_transfer_pair(
+// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?xf32>,
+// CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]]: vector<4xi1>,
+// CHECK: %[[READ:.*]] = vector.mask %[[MASK]] {{.*}}transfer_read %[[MEM]]
+// CHECK: %[[FOR:.*]] = scf.for {{.*}} iter_args(%[[IARG:.*]] = %[[READ]]) -> (vector<4xf32>) {
+// CHECK-NEXT: %[[USE:.*]] = "test.val_use"(%[[IARG]])
+// CHECK-NEXT: scf.yield %[[USE]]
+// CHECK-NEXT: }
+// CHECK: vector.mask %[[MASK]] {{.*}}transfer_write %[[FOR]], %[[MEM]]
+func.func @hoist_masked_transfer_pair(%mem: memref<?xf32>, %mask: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %r = vector.mask %mask { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.mask %mask { vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32> } : vector<4xi1>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A lone masked read is hoisted as its `vector.mask` op.
+
+// CHECK-LABEL: func.func @hoist_masked_singleton_read(
+// CHECK: %[[READ:.*]] = vector.mask %{{.*}}transfer_read
+// CHECK: scf.for
+// CHECK: "test.some_use"(%[[READ]])
+func.func @hoist_masked_singleton_read(%mem: memref<?xf32>, %mask: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %r = vector.mask %mask { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ "test.some_use"(%r) : (vector<4xf32>) -> ()
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A masked read paired with an unmasked write must not be hoisted: the boundary
+// masking would not be preserved.
+
+// CHECK-LABEL: func.func @negative_hoist_masked_read_unmasked_write(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.mask {{.*}}transfer_read
+// CHECK: vector.transfer_write
+// CHECK: }
+func.func @negative_hoist_masked_read_unmasked_write(%mem: memref<?xf32>, %mask: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %r = vector.mask %mask { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A masked pair carrying two distinct masks must not be hoisted.
+
+// CHECK-LABEL: func.func @negative_hoist_masked_pair_distinct_masks(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.mask {{.*}}transfer_read
+// CHECK: vector.mask {{.*}}transfer_write
+// CHECK: }
+func.func @negative_hoist_masked_pair_distinct_masks(%mem: memref<?xf32>, %mask0: vector<4xi1>, %mask1: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %r = vector.mask %mask0 { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.mask %mask1 { vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32> } : vector<4xi1>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// An unmasked read paired with a masked write must not be hoisted: the boundary
+// masking would not be preserved.
+
+// CHECK-LABEL: func.func @negative_hoist_unmasked_read_masked_write(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.transfer_read
+// CHECK: vector.mask {{.*}}transfer_write
+// CHECK: }
+func.func @negative_hoist_unmasked_read_masked_write(%mem: memref<?xf32>, %mask: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %r = vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.mask %mask { vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32> } : vector<4xi1>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A masked pair whose mask is recomputed in the loop must not be hoisted: the
+// mask is not loop-invariant.
+
+// CHECK-LABEL: func.func @negative_hoist_masked_pair_loop_variant_mask(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.mask {{.*}}transfer_read
+// CHECK: vector.mask {{.*}}transfer_write
+// CHECK: }
+func.func @negative_hoist_masked_pair_loop_variant_mask(%mem: memref<?xf32>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %mask = vector.create_mask %i : vector<4xi1>
+ %r = vector.mask %mask { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.mask %mask { vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32> } : vector<4xi1>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
+
+// -----
+
+// A masked read whose passthru is recomputed in the loop must not be hoisted:
+// the passthru is not loop-invariant, so the read value changes every iteration.
+
+// CHECK-LABEL: func.func @negative_hoist_masked_read_loop_variant_passthru(
+// CHECK: scf.for {{.*}} step %{{.*}} {
+// CHECK: vector.mask {{.*}}transfer_read
+// CHECK: vector.mask {{.*}}transfer_write
+// CHECK: }
+func.func @negative_hoist_masked_read_loop_variant_passthru(%mem: memref<?xf32>, %mask: vector<4xi1>, %lb: index, %ub: index, %step: index) {
+ %c0 = arith.constant 0 : index
+ %pad = arith.constant 0.0 : f32
+ scf.for %i = %lb to %ub step %step {
+ %idx = arith.index_cast %i : index to i32
+ %f = arith.sitofp %idx : i32 to f32
+ %pt = vector.broadcast %f : f32 to vector<4xf32>
+ %r = vector.mask %mask, %pt { vector.transfer_read %mem[%c0], %pad : memref<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+ %u = "test.val_use"(%r) : (vector<4xf32>) -> vector<4xf32>
+ vector.mask %mask { vector.transfer_write %u, %mem[%c0] : vector<4xf32>, memref<?xf32> } : vector<4xi1>
+ }
+ return
+}
+
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+ %0 = transform.structured.match ops{["func.func"]} in %arg1
+ : (!transform.any_op) -> !transform.any_op
+ transform.structured.hoist_redundant_vector_transfers %0
+ : (!transform.any_op) -> !transform.any_op
+ transform.yield
+ }
+}
More information about the Mlir-commits
mailing list