[Mlir-commits] [mlir] [mlir][vector] Fix crash on untraceable masks in getCompressedMaskOp (PR #207299)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Fri Jul 3 10:13:55 PDT 2026
https://github.com/marquisburg updated https://github.com/llvm/llvm-project/pull/207299
>From bb179a3054918920bf0f77f7d8b141d176e630de Mon Sep 17 00:00:00 2001
From: GhostDragonWasTook <marquis at marquiseco.co.za>
Date: Fri, 3 Jul 2026 02:40:25 +0200
Subject: [PATCH 1/2] [mlir][vector] Fix crash on untraceable masks in
getCompressedMaskOp
The loop tracing a mask back to its creation op assumed the chain
always ends at a create_mask/constant_mask/arith.constant, a
block-argument mask made it fall through to isa<> on a null op
(segfault), and any other unhandled defining op spun it forever.
Fixes #206928
---
.../Transforms/VectorEmulateNarrowType.cpp | 8 +++++-
.../emulate-narrow-type-unsupported.mlir | 28 +++++++++++++++++++
2 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index 9faaebdcf8f35..e37296e50fb29 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -98,10 +98,16 @@ static FailureOr<Operation *> getCompressedMaskOp(OpBuilder &rewriter,
if (auto extractOp = dyn_cast<vector::ExtractOp>(maskOp)) {
maskOp = extractOp.getSource().getDefiningOp();
extractOps.push_back(extractOp);
+ } else {
+ // Unsupported mask-defining op (e.g. a block argument, which has no
+ // defining op, or an op we cannot trace through). Bail out rather than
+ // looping forever or dereferencing a null op below.
+ break;
}
}
- if (!isa<arith::ConstantOp, vector::CreateMaskOp, vector::ConstantMaskOp>(
+ if (!maskOp ||
+ !isa<arith::ConstantOp, vector::CreateMaskOp, vector::ConstantMaskOp>(
maskOp))
return failure();
diff --git a/mlir/test/Dialect/Vector/emulate-narrow-type-unsupported.mlir b/mlir/test/Dialect/Vector/emulate-narrow-type-unsupported.mlir
index a5a6fc4acfe10..74f72ed5a5d9c 100644
--- a/mlir/test/Dialect/Vector/emulate-narrow-type-unsupported.mlir
+++ b/mlir/test/Dialect/Vector/emulate-narrow-type-unsupported.mlir
@@ -109,3 +109,31 @@ func.func @vector_maskedstore_2d_i8_negative(%arg0: index, %arg1: index, %arg2:
// CHECK-LABEL: func @vector_maskedstore_2d_i8_negative
// CHECK: memref.alloc() : memref<3x8xi8>
// CHECK-NOT: i32
+
+// -----
+
+// The mask is a block argument, so its defining op cannot be traced back to a
+// supported mask-creation op. This must gracefully bail out instead of crashing
+// (see llvm/llvm-project#206928).
+
+func.func @vector_maskedstore_i8_block_arg_mask_negative(%arg0: memref<?xi8>, %mask: vector<8xi1>, %value: vector<8xi8>) {
+ %c0 = arith.constant 0 : index
+ vector.maskedstore %arg0[%c0], %mask, %value : memref<?xi8>, vector<8xi1>, vector<8xi8>
+ return
+}
+
+// CHECK-LABEL: func @vector_maskedstore_i8_block_arg_mask_negative
+// CHECK: vector.maskedstore
+
+// -----
+
+// As above, but for `vector.maskedload`.
+
+func.func @vector_maskedload_i8_block_arg_mask_negative(%arg0: memref<?xi8>, %mask: vector<8xi1>, %passthru: vector<8xi8>) -> vector<8xi8> {
+ %c0 = arith.constant 0 : index
+ %0 = vector.maskedload %arg0[%c0], %mask, %passthru : memref<?xi8>, vector<8xi1>, vector<8xi8> into vector<8xi8>
+ return %0 : vector<8xi8>
+}
+
+// CHECK-LABEL: func @vector_maskedload_i8_block_arg_mask_negative
+// CHECK: vector.maskedload
>From 8ce3d9fdeae633ea27cca1ab59c1625d3ca0e465 Mon Sep 17 00:00:00 2001
From: GhostDragonWasTook <marquis at marquiseco.co.za>
Date: Fri, 3 Jul 2026 18:52:14 +0200
Subject: [PATCH 2/2] [mlir][vector] Fix unaligned-offset miscompiles in
narrow-type emulation
ConvertVectorLoad, ConvertVectorMaskedLoad, and ConvertVectorTransferRead
treated a vector size divisble by the container size as proof that the access
starts on a container boundary, and ConvertVectorMaskedStore never looked
at the offset at all. Any size-divisible access at an unaligned offset
(e.g. vector<8xi4> at index 1) was silently read or written one sub-element early.
Always fold the real intra-container offset: loads reuse the existing static/dynamic
extraction paths, and maskedstore bails out unless the offset provably folds to zero,
the same contract I saw in #189235 for vector.store.
---
.../Transforms/VectorEmulateNarrowType.cpp | 54 +--
...atten-memref-and-emulate-narrow-types.mlir | 11 +-
...e-narrow-type-unaligned-dynamic-store.mlir | 49 +++
.../vector-emulate-narrow-type-unaligned.mlir | 70 ++++
.../Vector/vector-emulate-narrow-type.mlir | 312 ++++++++++++------
5 files changed, 369 insertions(+), 127 deletions(-)
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index e37296e50fb29..54c79ebce7289 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -910,6 +910,18 @@ struct ConvertVectorMaskedStore final
Value linearizedIndices =
getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndicesOfr);
+ // The load-select-store sequence below assumes the store starts at a
+ // container element boundary. A non-zero intra-container offset would
+ // require shifting both the mask and the value to store; that is not
+ // implemented yet, so bail out to avoid storing at the wrong offset.
+ // Note: a size divisible by `emulatedPerContainerElem` does NOT imply
+ // such alignment - the offset depends on the (possibly dynamic) indices.
+ std::optional<int64_t> intraDataOffset =
+ getConstantIntValue(linearizedInfo.intraDataOffset);
+ if (!intraDataOffset || *intraDataOffset != 0)
+ return rewriter.notifyMatchFailure(
+ op, "unaligned or dynamically-offset maskedstore is not supported");
+
// Load the whole data and use arith.select to handle the corner cases.
//
// As an example, for this masked store of i4 values:
@@ -1042,8 +1054,6 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
// compile time as they must be constants.
auto origElements = op.getVectorType().getNumElements();
- // Note, per-element-alignment was already verified above.
- bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
auto stridedMetadata =
memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
@@ -1058,9 +1068,11 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
stridedMetadata.getConstifiedMixedStrides(),
getAsOpFoldResult(adaptor.getIndices()));
+ // Note: a size divisible by `emulatedPerContainerElem` does NOT imply the
+ // load starts at a container element boundary - the intra-container
+ // offset depends on the (possibly dynamic) indices.
std::optional<int64_t> foldedIntraVectorOffset =
- isDivisibleInSize ? 0
- : getConstantIntValue(linearizedInfo.intraDataOffset);
+ getConstantIntValue(linearizedInfo.intraDataOffset);
// Always load enough elements which can cover the original elements.
int64_t maxintraDataOffset =
@@ -1077,7 +1089,9 @@ struct ConvertVectorLoad final : OpConversionPattern<vector::LoadOp> {
result = dynamicallyExtractSubVector(
rewriter, loc, dyn_cast<TypedValue<VectorType>>(result), resultVector,
linearizedInfo.intraDataOffset, origElements);
- } else if (!isDivisibleInSize) {
+ } else {
+ // This is a no-op if the offset is zero and all loaded elements are
+ // used.
result = staticallyExtractSubvector(
rewriter, loc, result, *foldedIntraVectorOffset, origElements);
}
@@ -1167,8 +1181,6 @@ struct ConvertVectorMaskedLoad final
// subvector at the proper offset after bit-casting.
auto origType = op.getVectorType();
auto origElements = origType.getNumElements();
- // Note, per-element-alignment was already verified above.
- bool isDivisibleInSize = origElements % emulatedPerContainerElem == 0;
auto stridedMetadata =
memref::ExtractStridedMetadataOp::create(rewriter, loc, op.getBase());
@@ -1182,9 +1194,11 @@ struct ConvertVectorMaskedLoad final
stridedMetadata.getConstifiedMixedStrides(),
getAsOpFoldResult(adaptor.getIndices()));
+ // Note: a size divisible by `emulatedPerContainerElem` does NOT imply the
+ // load starts at a container element boundary - the intra-container
+ // offset depends on the (possibly dynamic) indices.
std::optional<int64_t> foldedIntraVectorOffset =
- isDivisibleInSize ? 0
- : getConstantIntValue(linearizedInfo.intraDataOffset);
+ getConstantIntValue(linearizedInfo.intraDataOffset);
int64_t maxIntraDataOffset =
foldedIntraVectorOffset.value_or(emulatedPerContainerElem - 1);
@@ -1208,7 +1222,8 @@ struct ConvertVectorMaskedLoad final
passthru = dynamicallyInsertSubVector(
rewriter, loc, passthru, emptyVector, linearizedInfo.intraDataOffset,
origElements);
- } else if (!isDivisibleInSize) {
+ } else {
+ // No-op when the offset is zero and the sizes match.
passthru = staticallyInsertSubvector(rewriter, loc, passthru, emptyVector,
*foldedIntraVectorOffset);
}
@@ -1237,7 +1252,8 @@ struct ConvertVectorMaskedLoad final
mask = dynamicallyInsertSubVector(rewriter, loc, mask, emptyMask,
linearizedInfo.intraDataOffset,
origElements);
- } else if (!isDivisibleInSize) {
+ } else {
+ // No-op when the offset is zero and the sizes match.
mask = staticallyInsertSubvector(rewriter, loc, op.getMask(), emptyMask,
*foldedIntraVectorOffset);
}
@@ -1248,7 +1264,8 @@ struct ConvertVectorMaskedLoad final
result = dynamicallyExtractSubVector(
rewriter, loc, result, op.getPassThru(),
linearizedInfo.intraDataOffset, origElements);
- } else if (!isDivisibleInSize) {
+ } else {
+ // No-op when the offset is zero and all loaded elements are used.
result = staticallyExtractSubvector(
rewriter, loc, result, *foldedIntraVectorOffset, origElements);
}
@@ -1325,10 +1342,6 @@ struct ConvertVectorTransferRead final
auto origElements = op.getVectorType().getNumElements();
- // Note, per-element-alignment was already verified above.
- bool isDivisibleInSize =
- fitsInMultiByteContainerTy(op.getVectorType(), containerElemTy);
-
// Pad the padding value with 0s on the left. These bits are discarded and
// thus their values don't matter.
Value padding = adaptor.getPadding();
@@ -1355,9 +1368,11 @@ struct ConvertVectorTransferRead final
stridedMetadata.getConstifiedMixedStrides(),
getAsOpFoldResult(adaptor.getIndices()));
+ // Note: a size divisible by `emulatedPerContainerElem` does NOT imply the
+ // read starts at a container element boundary - the intra-container
+ // offset depends on the (possibly dynamic) indices.
std::optional<int64_t> foldedIntraVectorOffset =
- isDivisibleInSize ? 0
- : getConstantIntValue(linearizedInfo.intraDataOffset);
+ getConstantIntValue(linearizedInfo.intraDataOffset);
int64_t maxIntraDataOffset =
foldedIntraVectorOffset.value_or(emulatedPerContainerElem - 1);
@@ -1382,7 +1397,8 @@ struct ConvertVectorTransferRead final
result = dynamicallyExtractSubVector(rewriter, loc, bitCast, zeros,
linearizedInfo.intraDataOffset,
origElements);
- } else if (!isDivisibleInSize) {
+ } else {
+ // No-op when the offset is zero and all read elements are used.
result = staticallyExtractSubvector(
rewriter, loc, result, *foldedIntraVectorOffset, origElements);
}
diff --git a/mlir/test/Dialect/Vector/flatten-memref-and-emulate-narrow-types.mlir b/mlir/test/Dialect/Vector/flatten-memref-and-emulate-narrow-types.mlir
index ef253624fe0b4..38f4d8b0a6ff0 100644
--- a/mlir/test/Dialect/Vector/flatten-memref-and-emulate-narrow-types.mlir
+++ b/mlir/test/Dialect/Vector/flatten-memref-and-emulate-narrow-types.mlir
@@ -31,10 +31,14 @@ func.func @vector_maskedload_2d_i4(%arg0: index, %passthru: vector<8xi4>) -> vec
// -----
+// Note: the column index must be a constant so that the intra-byte offset of
+// the store provably folds to 0 (a maskedstore at an offset that cannot be
+// proven to be byte-aligned cannot be emulated and would fail to legalize).
func.func @vector_maskedstore_2d_i4(%arg0: index, %value: vector<8xi4>) {
%0 = memref.alloc() : memref<4x8xi4>
+ %c0 = arith.constant 0 : index
%mask = vector.constant_mask [5] : vector<8xi1>
- vector.maskedstore %0[%arg0, %arg0], %mask, %value :
+ vector.maskedstore %0[%arg0, %c0], %mask, %value :
memref<4x8xi4>, vector<8xi1>, vector<8xi4>
return
}
@@ -54,9 +58,12 @@ func.func @vector_store_2d_i4(%arg0: index, %value: vector<8xi4>) {
// -----
+// Note: the column index is a constant so that the intra-byte offset of the
+// read provably folds to 0 and no extra byte needs to be read.
func.func @vector_transfer_read_2d_i4(%arg0: index, %padding: i4) -> vector<8xi4> {
%0 = memref.alloc() : memref<4x8xi4>
- %1 = vector.transfer_read %0[%arg0, %arg0], %padding {in_bounds = [true]} : memref<4x8xi4>, vector<8xi4>
+ %c0 = arith.constant 0 : index
+ %1 = vector.transfer_read %0[%arg0, %c0], %padding {in_bounds = [true]} : memref<4x8xi4>, vector<8xi4>
return %1 : vector<8xi4>
}
// CHECK-LABEL: func @vector_transfer_read_2d_i4(
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-dynamic-store.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-dynamic-store.mlir
index 518d825234502..ba2293c6321e2 100644
--- a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-dynamic-store.mlir
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned-dynamic-store.mlir
@@ -20,3 +20,52 @@ func.func @vector_store_i4_dynamic_memref(%arg0: vector<8xi4>, %dim0: index,
vector.store %arg0, %src[%idx0, %idx1] : memref<?x?xi4>, vector<8xi4>
return
}
+
+// -----
+
+// Dynamic sub-byte vector.maskedstore offsets cannot be treated as
+// byte-aligned either: the store could start in the middle of a byte and the
+// emulation (which read-modify-writes whole bytes) would write to the wrong
+// bits.
+
+func.func @vector_maskedstore_i4_dynamic_col(%idx1: index, %idx2: index,
+ %num_elements_to_store: index,
+ %value: vector<8xi4>) {
+ %0 = memref.alloc() : memref<3x8xi4>
+ %mask = vector.create_mask %num_elements_to_store : vector<8xi1>
+ // expected-error @below {{failed to legalize operation 'vector.maskedstore' that was explicitly marked illegal}}
+ vector.maskedstore %0[%idx1, %idx2], %mask, %value :
+ memref<3x8xi4>, vector<8xi1>, vector<8xi4>
+ return
+}
+
+// -----
+
+func.func @vector_maskedstore_i4_constant_mask_dynamic_col(%idx1: index,
+ %idx2: index,
+ %value: vector<8xi4>) {
+ %0 = memref.alloc() : memref<3x8xi4>
+ %mask = vector.constant_mask [4] : vector<8xi1>
+ // expected-error @below {{failed to legalize operation 'vector.maskedstore' that was explicitly marked illegal}}
+ vector.maskedstore %0[%idx1, %idx2], %mask, %value :
+ memref<3x8xi4>, vector<8xi1>, vector<8xi4>
+ return
+}
+
+// -----
+
+// A maskedstore at a static but non-container-aligned offset (an odd i4 index
+// gives intra-byte offset 1) must also be rejected - emulating it with the
+// compressed mask would silently store at the wrong bits (the intra-byte
+// offset used to be dropped; see the discussion in llvm/llvm-project#206928).
+
+func.func @vector_maskedstore_i4_unaligned_static_offset(%num_elements_to_store: index,
+ %value: vector<8xi4>) {
+ %c1 = arith.constant 1 : index
+ %0 = memref.alloc() : memref<9xi4>
+ %mask = vector.create_mask %num_elements_to_store : vector<8xi1>
+ // expected-error @below {{failed to legalize operation 'vector.maskedstore' that was explicitly marked illegal}}
+ vector.maskedstore %0[%c1], %mask, %value :
+ memref<9xi4>, vector<8xi1>, vector<8xi4>
+ return
+}
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
index bec7736b90973..60ceecf88ddaf 100644
--- a/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type-unaligned.mlir
@@ -586,3 +586,73 @@ func.func @vector_store_i2_2d_const_nonzero_col(%arg0: vector<4xi2>) {
// and one for byte 1 (element at position [0]).
// CHECK: memref.generic_atomic_rmw %[[ALLOC]][%[[C0]]]
// CHECK: memref.generic_atomic_rmw %[[ALLOC]][{{.+}}]
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Size-divisible vectors at non-container-aligned static offsets.
+///
+/// A vector length divisible by the number of emulated elements per container
+/// element does NOT imply the access starts at a container element boundary.
+/// These used to be emulated as if aligned, silently reading shifted data
+/// (see the discussion in llvm/llvm-project#206928).
+///----------------------------------------------------------------------------------------
+
+func.func @vector_load_i4_unaligned_static_offset() -> vector<8xi4> {
+ %c1 = arith.constant 1 : index
+ %0 = memref.alloc() : memref<9xi4>
+ %1 = vector.load %0[%c1] : memref<9xi4>, vector<8xi4>
+ return %1 : vector<8xi4>
+}
+// CHECK: func @vector_load_i4_unaligned_static_offset(
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<5xi8>
+// CHECK: %[[C0:.+]] = arith.constant 0 : index
+// CHECK: %[[LOAD:.+]] = vector.load %[[ALLOC]][%[[C0]]] : memref<5xi8>, vector<5xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[SLICE:.+]] = vector.extract_strided_slice %[[BITCAST]] {offsets = [1], sizes = [8], strides = [1]} : vector<10xi4> to vector<8xi4>
+// CHECK: return %[[SLICE]]
+
+// -----
+
+func.func @vector_maskedload_i4_unaligned_static_offset(%num_elems: index, %passthru: vector<8xi4>) -> vector<8xi4> {
+ %c1 = arith.constant 1 : index
+ %0 = memref.alloc() : memref<9xi4>
+ %mask = vector.create_mask %num_elems : vector<8xi1>
+ %1 = vector.maskedload %0[%c1], %mask, %passthru : memref<9xi4>, vector<8xi1>, vector<8xi4> into vector<8xi4>
+ return %1 : vector<8xi4>
+}
+// The compressed mask accounts for the front offset of 1: ceil((n + 1) / 2).
+// CHECK: #[[MASK_MAP:.+]] = affine_map<()[s0] -> ((s0 + 1) ceildiv 2)>
+// CHECK: func @vector_maskedload_i4_unaligned_static_offset(
+// CHECK-SAME: %[[NUM_ELEMS:[a-zA-Z0-9]+]]: index, %[[PASSTHRU:[a-zA-Z0-9]+]]: vector<8xi4>)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<5xi8>
+// CHECK: %[[ORIG_MASK:.+]] = vector.create_mask %[[NUM_ELEMS]] : vector<8xi1>
+// CHECK: %[[MASK_IDX:.+]] = affine.apply #[[MASK_MAP]]()[%[[NUM_ELEMS]]]
+// CHECK: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<5xi1>
+// CHECK: %[[NEW_PASSTHRU:.+]] = vector.insert_strided_slice %[[PASSTHRU]], %{{.+}} {offsets = [1], strides = [1]} : vector<8xi4> into vector<10xi4>
+// CHECK: %[[PASSTHRU_BITCAST:.+]] = vector.bitcast %[[NEW_PASSTHRU]] : vector<10xi4> to vector<5xi8>
+// CHECK: %[[C0:.+]] = arith.constant 0 : index
+// CHECK: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[C0]]], %[[NEW_MASK]], %[[PASSTHRU_BITCAST]] : memref<5xi8>, vector<5xi1>, vector<5xi8> into vector<5xi8>
+// CHECK: %[[LOAD_BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[SHIFTED_MASK:.+]] = vector.insert_strided_slice %[[ORIG_MASK]], %{{.+}} {offsets = [1], strides = [1]} : vector<8xi1> into vector<10xi1>
+// CHECK: %[[SELECT:.+]] = arith.select %[[SHIFTED_MASK]], %[[LOAD_BITCAST]], %[[NEW_PASSTHRU]] : vector<10xi1>, vector<10xi4>
+// CHECK: %[[SLICE:.+]] = vector.extract_strided_slice %[[SELECT]] {offsets = [1], sizes = [8], strides = [1]} : vector<10xi4> to vector<8xi4>
+// CHECK: return %[[SLICE]]
+
+// -----
+
+func.func @vector_transfer_read_i4_unaligned_static_offset(%padding: i4) -> vector<8xi4> {
+ %c1 = arith.constant 1 : index
+ %0 = memref.alloc() : memref<9xi4>
+ %1 = vector.transfer_read %0[%c1], %padding {in_bounds = [true]} : memref<9xi4>, vector<8xi4>
+ return %1 : vector<8xi4>
+}
+// CHECK: func @vector_transfer_read_i4_unaligned_static_offset(
+// CHECK-SAME: %[[PADDING:[a-zA-Z0-9]+]]: i4)
+// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<5xi8>
+// CHECK: %[[PAD_I8:.+]] = arith.extui %[[PADDING]] : i4 to i8
+// CHECK: %[[C0:.+]] = arith.constant 0 : index
+// CHECK: %[[READ:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]]], %[[PAD_I8]] : memref<5xi8>, vector<5xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[READ]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[SLICE:.+]] = vector.extract_strided_slice %[[BITCAST]] {offsets = [1], sizes = [8], strides = [1]} : vector<10xi4> to vector<8xi4>
+// CHECK: return %[[SLICE]]
diff --git a/mlir/test/Dialect/Vector/vector-emulate-narrow-type.mlir b/mlir/test/Dialect/Vector/vector-emulate-narrow-type.mlir
index 72b355e0fed65..af31ac62f859e 100644
--- a/mlir/test/Dialect/Vector/vector-emulate-narrow-type.mlir
+++ b/mlir/test/Dialect/Vector/vector-emulate-narrow-type.mlir
@@ -17,14 +17,20 @@ func.func @vector_load_i8(%arg1: index, %arg2: index) -> vector<4xi8> {
// CHECK-NEXT: [[L:%.+]] = vector.load %[[ALLOC]][%[[ARG0]], %[[ARG1]]] : memref<3x4xi8>, vector<4xi8>
// CHECK-NEXT: return
-// CHECK32: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 4)>
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded and the target elements are extracted
+// dynamically.
+// CHECK32-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 4)>
+// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 4)>
// CHECK32: func @vector_load_i8(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[VECLOAD:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<1xi32>
-// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VECLOAD]] : vector<1xi32> to vector<4xi8>
-// CHECK32: return %[[VEC_I4]]
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK32: %[[VECLOAD:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<2xi32>
+// CHECK32: %[[VEC_I8:.+]] = vector.bitcast %[[VECLOAD]] : vector<2xi32> to vector<8xi8>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_I8]][%[[FRONT]]] : i8 from vector<8xi8>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i8 into vector<4xi8>
// -----
@@ -35,21 +41,32 @@ func.func @vector_load_i4(%arg1: index, %arg2: index) -> vector<3x8xi4> {
%2 = vector.insert %1, %cst [0] : vector<8xi4> into vector<3x8xi4>
return %2 : vector<3x8xi4>
}
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded and the target elements are extracted
+// dynamically.
// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 2)>
// CHECK: func @vector_load_i4
// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<12xi8>, vector<4xi8>
-// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<4xi8> to vector<8xi4>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<12xi8>, vector<5xi8>
+// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// CHECK32-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
+// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 8)>
// CHECK32: func @vector_load_i4
// CHECK32-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<1xi32>
-// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<1xi32> to vector<8xi4>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<2xi32>
+// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<2xi32> to vector<16xi4>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// -----
@@ -60,21 +77,32 @@ func.func @vector_load_f4(%arg1: index, %arg2: index) -> vector<3x8xf4E2M1FN> {
%2 = vector.insert %1, %cst [0] : vector<8xf4E2M1FN> into vector<3x8xf4E2M1FN>
return %2 : vector<3x8xf4E2M1FN>
}
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded and the target elements are extracted
+// dynamically.
// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 2)>
// CHECK: func @vector_load_f4
// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<12xi8>, vector<4xi8>
-// CHECK: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<4xi8> to vector<8xf4E2M1FN>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<12xi8>, vector<5xi8>
+// CHECK: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<5xi8> to vector<10xf4E2M1FN>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC_F4]][%[[FRONT]]] : f4E2M1FN from vector<10xf4E2M1FN>
+// CHECK-COUNT-8: vector.insert {{.+}} : f4E2M1FN into vector<8xf4E2M1FN>
// CHECK32-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
+// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 8)>
// CHECK32: func @vector_load_f4
// CHECK32-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<1xi32>
-// CHECK32: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<1xi32> to vector<8xf4E2M1FN>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<3xi32>, vector<2xi32>
+// CHECK32: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<2xi32> to vector<16xf4E2M1FN>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_F4]][%[[FRONT]]] : f4E2M1FN from vector<16xf4E2M1FN>
+// CHECK32-COUNT-8: vector.insert {{.+}} : f4E2M1FN into vector<8xf4E2M1FN>
// -----
@@ -83,8 +111,11 @@ func.func @vector_load_i4_dynamic(%arg0 : index, %arg1 : index, %arg2 : index, %
%1 = vector.load %0[%arg2, %arg3] : memref<?x?xi4>, vector<8xi4>
return %1 : vector<8xi4>
}
+// The intra-container offset of the load is dynamic, so one extra container
+// element is loaded and the target elements are extracted dynamically.
// CHECK-DAG: #[[MAP0:.+]] = affine_map<()[s0, s1] -> ((s0 * s1) floordiv 2, s0 floordiv 2)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> ((s2 + s0 * s1) floordiv 2)>
+// CHECK-DAG: #[[MAP2:.+]] = affine_map<()[s0, s1, s2] -> (s0 * s1 + s2 - ((s2 + s0 * s1) floordiv 2) * 2)>
// CHECK: func.func @vector_load_i4_dynamic(
// CHECK-SAME: %[[ARG0:[a-zA-Z0-9_]+]]: index
// CHECK-SAME: %[[ARG1:[a-zA-Z0-9_]+]]: index
@@ -93,11 +124,15 @@ func.func @vector_load_i4_dynamic(%arg0 : index, %arg1 : index, %arg2 : index, %
// CHECK: %[[SIZE:.+]] = affine.max #[[MAP0]]()[%[[ARG1]], %[[ARG0]]]
// CHECK: %[[ALLOC:.+]] = memref.alloc(%[[SIZE]]) : memref<?xi8>
// CHECK: %[[INDEX:.+]] = affine.apply #[[MAP1]]()[%[[ARG2]], %[[ARG1]], %[[ARG3]]]
-// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<?xi8>, vector<4xi8>
-// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<4xi8> to vector<8xi4>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[MAP2]]()[%[[ARG2]], %[[ARG1]], %[[ARG3]]]
+// CHECK: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<?xi8>, vector<5xi8>
+// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// CHECK32-DAG: #[[MAP0:.+]] = affine_map<()[s0, s1] -> ((s0 * s1) floordiv 8, s0 floordiv 8)>
// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1, s2] -> ((s2 + s0 * s1) floordiv 8)>
+// CHECK32-DAG: #[[MAP2:.+]] = affine_map<()[s0, s1, s2] -> (s0 * s1 + s2 - ((s2 + s0 * s1) floordiv 8) * 8)>
// CHECK32: func.func @vector_load_i4_dynamic(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9_]+]]: index
// CHECK32-SAME: %[[ARG1:[a-zA-Z0-9_]+]]: index
@@ -106,8 +141,11 @@ func.func @vector_load_i4_dynamic(%arg0 : index, %arg1 : index, %arg2 : index, %
// CHECK32: %[[SIZE:.+]] = affine.max #[[MAP0]]()[%[[ARG1]], %[[ARG0]]]
// CHECK32: %[[ALLOC:.+]] = memref.alloc(%[[SIZE]]) : memref<?xi32>
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP1]]()[%[[ARG2]], %[[ARG1]], %[[ARG3]]]
-// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<?xi32>, vector<1xi32>
-// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<1xi32> to vector<8xi4>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP2]]()[%[[ARG2]], %[[ARG1]], %[[ARG3]]]
+// CHECK32: %[[VEC:.+]] = vector.load %[[ALLOC]][%[[INDEX]]] : memref<?xi32>, vector<2xi32>
+// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<2xi32> to vector<16xi4>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// -----
@@ -122,25 +160,36 @@ func.func @vector_transfer_read_i4(%arg1: index, %arg2: index) -> vector<8xi4> {
memref<3x8xi4>, vector<8xi4>
return %1 : vector<8xi4>
}
+// The intra-container offset of the read is dynamic (it depends on %arg2), so
+// one extra container element is read and the target elements are extracted
+// dynamically.
// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 2)>
// CHECK: func @vector_transfer_read_i4
// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK: %[[CONST:.+]] = arith.constant 0 : i4
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[PAD:.+]] = arith.extui %[[CONST]] : i4 to i8
// CHECK: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<12xi8>, vector<4xi8>
-// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<4xi8> to vector<8xi4>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<12xi8>, vector<5xi8>
+// CHECK: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<5xi8> to vector<10xi4>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// CHECK32-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
+// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 8)>
// CHECK32: func @vector_transfer_read_i4
// CHECK32-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK32: %[[CONST:.+]] = arith.constant 0 : i4
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[PAD:.+]] = arith.extui %[[CONST]] : i4 to i32
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<3xi32>, vector<1xi32>
-// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<1xi32> to vector<8xi4>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK32: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<3xi32>, vector<2xi32>
+// CHECK32: %[[VEC_I4:.+]] = vector.bitcast %[[VEC]] : vector<2xi32> to vector<16xi4>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_I4]][%[[FRONT]]] : i4 from vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// -----
@@ -151,7 +200,11 @@ func.func @vector_transfer_read_f4(%arg1: index, %arg2: index) -> vector<8xf4E2M
memref<3x8xf4E2M1FN>, vector<8xf4E2M1FN>
return %1 : vector<8xf4E2M1FN>
}
+// The intra-container offset of the read is dynamic (it depends on %arg2), so
+// one extra container element is read and the target elements are extracted
+// dynamically.
// CHECK-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 2)>
// CHECK: func @vector_transfer_read_f4
// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK: %[[CONST:.+]] = arith.constant 0.{{0+}}e+00 : f4E2M1FN
@@ -159,10 +212,14 @@ func.func @vector_transfer_read_f4(%arg1: index, %arg2: index) -> vector<8xf4E2M
// CHECK: %[[BC:.+]] = arith.bitcast %[[CONST]] : f4E2M1FN to i4
// CHECK: %[[PAD:.+]] = arith.extui %[[BC]] : i4 to i8
// CHECK: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<12xi8>, vector<4xi8>
-// CHECK: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<4xi8> to vector<8xf4E2M1FN>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<12xi8>, vector<5xi8>
+// CHECK: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<5xi8> to vector<10xf4E2M1FN>
+// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC_F4]][%[[FRONT]]] : f4E2M1FN from vector<10xf4E2M1FN>
+// CHECK-COUNT-8: vector.insert {{.+}} : f4E2M1FN into vector<8xf4E2M1FN>
// CHECK32-DAG: #[[MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
+// CHECK32-DAG: #[[MAP1:.+]] = affine_map<()[s0] -> (s0 mod 8)>
// CHECK32: func @vector_transfer_read_f4
// CHECK32-SAME: (%[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index)
// CHECK32: %[[CONST:.+]] = arith.constant 0.{{0+}}e+00 : f4E2M1FN
@@ -170,8 +227,11 @@ func.func @vector_transfer_read_f4(%arg1: index, %arg2: index) -> vector<8xf4E2M
// CHECK32: %[[BC:.+]] = arith.bitcast %[[CONST]] : f4E2M1FN to i4
// CHECK32: %[[PAD:.+]] = arith.extui %[[BC]] : i4 to i32
// CHECK32: %[[INDEX:.+]] = affine.apply #[[MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<3xi32>, vector<1xi32>
-// CHECK32: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<1xi32> to vector<8xf4E2M1FN>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[MAP1]]()[%[[ARG1]]]
+// CHECK32: %[[VEC:.+]] = vector.transfer_read %[[ALLOC]][%[[INDEX]]], %[[PAD]] : memref<3xi32>, vector<2xi32>
+// CHECK32: %[[VEC_F4:.+]] = vector.bitcast %[[VEC]] : vector<2xi32> to vector<16xf4E2M1FN>
+// CHECK32: %[[EXTRACT:.+]] = vector.extract %[[VEC_F4]][%[[FRONT]]] : f4E2M1FN from vector<16xf4E2M1FN>
+// CHECK32-COUNT-8: vector.insert {{.+}} : f4E2M1FN into vector<8xf4E2M1FN>
// -----
@@ -196,22 +256,29 @@ func.func @vector_maskedload_i8(%arg1: index, %arg2: index, %arg3: index, %passt
// CHECK-SAME: memref<3x4xi8>, vector<4xi1>, vector<4xi8> into vector<4xi8>
// CHECK-NEXT: return
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded; the mask, the passthru and the result
+// are shifted dynamically by the front offset.
// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 4)>
-// CHECK32-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> (s0 ceildiv 4)>
+// CHECK32-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 4)>
+// CHECK32-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> ((s0 + 3) ceildiv 4)>
// CHECK32: func @vector_maskedload_i8(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[ARG2:[a-zA-Z0-9]+]]: index, %[[ARG3:[a-zA-Z0-9]+]]: vector<4xi8>)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.create_mask %[[ARG2]] : vector<4xi1>
// CHECK32: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
// CHECK32: %[[MASK_IDX:.+]] = affine.apply #[[MASK_IDX_MAP]]()[%[[ARG2]]]
-// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<1xi1>
-// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG3]] : vector<4xi8> to vector<1xi32>
+// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<2xi1>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i8 into vector<8xi8>
+// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<8xi8> to vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK32-SAME: memref<3xi32>, vector<1xi1>, vector<1xi32> into vector<1xi32>
-// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi32> to vector<4xi8>
-// CHECK32: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG3]] : vector<4xi1>, vector<4xi8>
-// CHECK32: return %[[SELECT]]
+// CHECK32-SAME: memref<3xi32>, vector<2xi1>, vector<2xi32> into vector<2xi32>
+// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<2xi32> to vector<8xi8>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i1 into vector<8xi1>
+// CHECK32: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<8xi1>, vector<8xi8>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i8 into vector<4xi8>
// -----
@@ -224,37 +291,50 @@ func.func @vector_maskedload_i4(%arg1: index, %arg2: index, %arg3: index, %passt
%2 = vector.insert %1, %cst [0] : vector<8xi4> into vector<3x8xi4>
return %2 : vector<3x8xi4>
}
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded; the mask, the passthru and the result
+// are shifted dynamically by the front offset.
// CHECK-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
-// CHECK-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> (s0 ceildiv 2)>
+// CHECK-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 2)>
+// CHECK-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> ((s0 + 1) ceildiv 2)>
// CHECK: func @vector_maskedload_i4(
// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[ARG2:[a-zA-Z0-9]+]]: index, %[[ARG3:[a-zA-Z0-9]+]]: vector<8xi4>)
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[ORIG_MASK:.+]] = vector.create_mask %[[ARG2]] : vector<8xi1>
// CHECK: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
+// CHECK: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
// CHECK: %[[MASK_IDX:.+]] = affine.apply #[[MASK_IDX_MAP]]()[%[[ARG2]]]
-// CHECK: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<4xi1>
-// CHECK: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG3]] : vector<8xi4> to vector<4xi8>
+// CHECK: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<5xi1>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<10xi4>
+// CHECK: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<10xi4> to vector<5xi8>
// CHECK: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK-SAME: memref<12xi8>, vector<4xi1>, vector<4xi8> into vector<4xi8>
-// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<4xi8> to vector<8xi4>
-// CHECK: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG3]] : vector<8xi1>, vector<8xi4>
+// CHECK-SAME: memref<12xi8>, vector<5xi1>, vector<5xi8> into vector<5xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<5xi8> to vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i1 into vector<10xi1>
+// CHECK: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<10xi1>, vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
-// CHECK32-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> (s0 ceildiv 8)>
+// CHECK32-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 8)>
+// CHECK32-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> ((s0 + 7) ceildiv 8)>
// CHECK32: func @vector_maskedload_i4(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[ARG2:[a-zA-Z0-9]+]]: index, %[[ARG3:[a-zA-Z0-9]+]]: vector<8xi4>)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.create_mask %[[ARG2]] : vector<8xi1>
// CHECK32: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
// CHECK32: %[[MASK_IDX:.+]] = affine.apply #[[MASK_IDX_MAP]]()[%[[ARG2]]]
-// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<1xi1>
-// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG3]] : vector<8xi4> to vector<1xi32>
+// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<2xi1>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<16xi4>
+// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<16xi4> to vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK32-SAME: memref<3xi32>, vector<1xi1>, vector<1xi32> into vector<1xi32>
-// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi32> to vector<8xi4>
-// CHECK32: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG3]] : vector<8xi1>, vector<8xi4>
+// CHECK32-SAME: memref<3xi32>, vector<2xi1>, vector<2xi32> into vector<2xi32>
+// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<2xi32> to vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i1 into vector<16xi1>
+// CHECK32: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<16xi1>, vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// -----
@@ -275,20 +355,28 @@ func.func @vector_maskedload_i8_constant_mask(%arg1: index, %arg2: index, %passt
// CHECK-SAME: memref<3x4xi8>, vector<4xi1>, vector<4xi8> into vector<4xi8>
// CHECK-NEXT: return
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded (with a pessimistic compressed mask);
+// the mask, the passthru and the result are shifted dynamically by the front
+// offset.
// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 4)>
+// CHECK32-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 4)>
// CHECK32: func @vector_maskedload_i8_constant_mask(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[ARG3:[a-zA-Z0-9]+]]: vector<4xi8>)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.constant_mask [2] : vector<4xi1>
// CHECK32: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [1] : vector<1xi1>
-// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG3]] : vector<4xi8> to vector<1xi32>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
+// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [2] : vector<2xi1>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i8 into vector<8xi8>
+// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<8xi8> to vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK32-SAME: memref<3xi32>, vector<1xi1>, vector<1xi32> into vector<1xi32>
-// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi32> to vector<4xi8>
-// CHECK32: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG3]] : vector<4xi1>, vector<4xi8>
-// CHECK32: return %[[SELECT]]
+// CHECK32-SAME: memref<3xi32>, vector<2xi1>, vector<2xi32> into vector<2xi32>
+// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<2xi32> to vector<8xi8>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i1 into vector<8xi1>
+// CHECK32: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<8xi1>, vector<8xi8>
+// CHECK32-COUNT-4: vector.insert {{.+}} : i8 into vector<4xi8>
// -----
@@ -301,33 +389,47 @@ func.func @vector_maskedload_i4_constant_mask(%arg1: index, %arg2: index, %passt
%2 = vector.insert %1, %cst [0] : vector<8xi4> into vector<3x8xi4>
return %2 : vector<3x8xi4>
}
+// The intra-container offset of the load is dynamic (it depends on %arg2), so
+// one extra container element is loaded (with a pessimistic compressed mask);
+// the mask, the passthru and the result are shifted dynamically by the front
+// offset.
// CHECK-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
+// CHECK-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 2)>
// CHECK: func @vector_maskedload_i4_constant_mask(
// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[ARG2:[a-zA-Z0-9]+]]: vector<8xi4>)
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[ORIG_MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
// CHECK: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK: %[[NEW_MASK:.+]] = vector.constant_mask [2] : vector<4xi1>
-// CHECK: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG2]] : vector<8xi4> to vector<4xi8>
+// CHECK: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
+// CHECK: %[[NEW_MASK:.+]] = vector.constant_mask [3] : vector<5xi1>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<10xi4>
+// CHECK: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<10xi4> to vector<5xi8>
// CHECK: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK-SAME: memref<12xi8>, vector<4xi1>, vector<4xi8> into vector<4xi8>
-// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<4xi8> to vector<8xi4>
-// CHECK: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG2]] : vector<8xi1>, vector<8xi4>
+// CHECK-SAME: memref<12xi8>, vector<5xi1>, vector<5xi8> into vector<5xi8>
+// CHECK: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<5xi8> to vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i1 into vector<10xi1>
+// CHECK: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<10xi1>, vector<10xi4>
+// CHECK-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
+// CHECK32-DAG: #[[FRONT_MAP:.+]] = affine_map<()[s0] -> (s0 mod 8)>
// CHECK32: func @vector_maskedload_i4_constant_mask(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]: index, %[[ARG1:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[ARG2:[a-zA-Z0-9]+]]: vector<8xi4>)
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
// CHECK32: %[[LD_IDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [1] : vector<1xi1>
-// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %[[ARG2]] : vector<8xi4> to vector<1xi32>
+// CHECK32: %[[FRONT:.+]] = affine.apply #[[FRONT_MAP]]()[%[[ARG1]]]
+// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [2] : vector<2xi1>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<16xi4>
+// CHECK32: %[[NEW_PASSTHRU:.+]] = vector.bitcast %{{.+}} : vector<16xi4> to vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LD_IDX]]], %[[NEW_MASK]], %[[NEW_PASSTHRU]] :
-// CHECK32-SAME: memref<3xi32>, vector<1xi1>, vector<1xi32> into vector<1xi32>
-// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<1xi32> to vector<8xi4>
-// CHECK32: %[[SELECT:.+]] = arith.select %[[ORIG_MASK]], %[[BITCAST]], %[[ARG2]] : vector<8xi1>, vector<8xi4>
+// CHECK32-SAME: memref<3xi32>, vector<2xi1>, vector<2xi32> into vector<2xi32>
+// CHECK32: %[[BITCAST:.+]] = vector.bitcast %[[LOAD]] : vector<2xi32> to vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i1 into vector<16xi1>
+// CHECK32: %[[SELECT:.+]] = arith.select %{{.+}}, %[[BITCAST]], %{{.+}} : vector<16xi1>, vector<16xi4>
+// CHECK32-COUNT-8: vector.insert {{.+}} : i4 into vector<8xi4>
// -----
@@ -533,34 +635,36 @@ func.func @vector_store_f4(%arg0: vector<8xf4E2M1FN>, %arg1: index, %arg2: index
/// vector.maskedstore
///----------------------------------------------------------------------------------------
-func.func @vector_maskedstore_i8(%arg0: index, %arg1: index, %arg2: index, %value: vector<8xi8>) {
+// Note: emulating a maskedstore requires a static, container-aligned offset
+// (a dynamic offset makes it impossible to prove alignment and the pattern
+// bails out), hence the constant indices.
+func.func @vector_maskedstore_i8(%arg0: index, %value: vector<8xi8>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
%0 = memref.alloc() : memref<3x8xi8>
- %mask = vector.create_mask %arg2 : vector<8xi1>
- vector.maskedstore %0[%arg0, %arg1], %mask, %value : memref<3x8xi8>, vector<8xi1>, vector<8xi8>
+ %mask = vector.create_mask %arg0 : vector<8xi1>
+ vector.maskedstore %0[%c1, %c0], %mask, %value : memref<3x8xi8>, vector<8xi1>, vector<8xi8>
return
}
// Expect no conversions, i8 is supported.
// CHECK: func @vector_maskedstore_i8(
// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]
-// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]]
-// CHECK-SAME: %[[ARG2:[a-zA-Z0-9]+]]
// CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]]
-// CHECK-NEXT: %[[ALLOC:.+]] = memref.alloc() : memref<3x8xi8>
-// CHECK-NEXT: %[[MASK:.+]] = vector.create_mask %[[ARG2]] : vector<8xi1>
-// CHECK-NEXT: vector.maskedstore %[[ALLOC]][%[[ARG0]], %[[ARG1]]], %[[MASK]], %[[VAL]]
+// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index
+// CHECK-DAG: %[[ALLOC:.+]] = memref.alloc() : memref<3x8xi8>
+// CHECK: %[[MASK:.+]] = vector.create_mask %[[ARG0]] : vector<8xi1>
+// CHECK-NEXT: vector.maskedstore %[[ALLOC]][%[[C1]], %[[C0]]], %[[MASK]], %[[VAL]]
// CHECK-NEXT: return
-// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 2 + s1 floordiv 4)>
// CHECK32-DAG: #[[MASK_IDX_MAP:.+]] = affine_map<()[s0] -> (s0 ceildiv 4)>
// CHECK32: func @vector_maskedstore_i8(
// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]
-// CHECK32-SAME: %[[ARG1:[a-zA-Z0-9]+]]
-// CHECK32-SAME: %[[ARG2:[a-zA-Z0-9]+]]
// CHECK32-SAME: %[[VAL:[a-zA-Z0-9]+]]
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<6xi32>
-// CHECK32: %[[ORIG_MASK:.+]] = vector.create_mask %[[ARG2]] : vector<8xi1>
-// CHECK32: %[[LIDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
-// CHECK32: %[[MASK_IDX:.+]] = affine.apply #[[MASK_IDX_MAP]]()[%[[ARG2]]]
+// CHECK32: %[[ORIG_MASK:.+]] = vector.create_mask %[[ARG0]] : vector<8xi1>
+// CHECK32: %[[LIDX:.+]] = arith.constant 2 : index
+// CHECK32: %[[MASK_IDX:.+]] = affine.apply #[[MASK_IDX_MAP]]()[%[[ARG0]]]
// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<2xi1>
// CHECK32: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LIDX]]], %[[NEW_MASK]], %[[PASS_THRU]]
@@ -571,29 +675,29 @@ func.func @vector_maskedstore_i8(%arg0: index, %arg1: index, %arg2: index, %valu
// -----
+// Note: emulating a maskedstore requires a static, container-aligned offset
+// (a dynamic offset makes it impossible to prove alignment and the pattern
+// bails out), hence the constant indices.
func.func @vector_maskedstore_i4(
- %idx1: index,
- %idx2: index,
%num_elements_to_store: index,
%value: vector<8xi4>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
%0 = memref.alloc() : memref<3x8xi4>
%mask = vector.create_mask %num_elements_to_store : vector<8xi1>
- vector.maskedstore %0[%idx1, %idx2], %mask, %value :
+ vector.maskedstore %0[%c1, %c0], %mask, %value :
memref<3x8xi4>, vector<8xi1>, vector<8xi4>
return
}
-// CHECK: #[[$ATTR_10:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
// CHECK: #[[$ATTR_11:.+]] = affine_map<()[s0] -> (s0 ceildiv 2)>
// CHECK: func.func @vector_maskedstore_i4(
-// CHECK-SAME: %[[IDX_1:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IDX_2:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[NUM_EL_TO_STORE:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[VAL_TO_STORE:[a-zA-Z0-9]+]]: vector<8xi4>) {
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[ORIG_MASK:.+]] = vector.create_mask %[[NUM_EL_TO_STORE]] : vector<8xi1>
-// CHECK: %[[LIDX:.+]] = affine.apply #[[$ATTR_10]]()[%[[IDX_1]], %[[IDX_2]]]
+// CHECK: %[[LIDX:.+]] = arith.constant 4 : index
// CHECK: %[[MASK_IDX:.+]] = affine.apply #[[$ATTR_11]]()[%[[NUM_EL_TO_STORE]]]
// CHECK: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<4xi1>
// CHECK: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<4xi8>
@@ -603,17 +707,14 @@ func.func @vector_maskedstore_i4(
// CHECK: %[[NEW_VAL:.+]] = vector.bitcast %[[SELECT]] : vector<8xi4> to vector<4xi8>
// CHECK: vector.maskedstore %[[ALLOC]]{{\[}}%[[LIDX]]], %[[NEW_MASK]], %[[NEW_VAL]] : memref<12xi8>, vector<4xi1>, vector<4xi8>
-// CHECK32: #[[$ATTR_17:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
// CHECK32: #[[$ATTR_18:.+]] = affine_map<()[s0] -> (s0 ceildiv 8)>
// CHECK32: func.func @vector_maskedstore_i4(
-// CHECK32-SAME: %[[IDX_1:[a-zA-Z0-9]+]]: index,
-// CHECK32-SAME: %[[IDX_2:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[NUM_EL_TO_STORE:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[VAL_TO_STORE:[a-zA-Z0-9]+]]: vector<8xi4>) {
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.create_mask %[[NUM_EL_TO_STORE]] : vector<8xi1>
-// CHECK32: %[[LIDX:.+]] = affine.apply #[[$ATTR_17]]()[%[[IDX_1]], %[[IDX_2]]]
+// CHECK32: %[[LIDX:.+]] = arith.constant 1 : index
// CHECK32: %[[MASK_IDX:.+]] = affine.apply #[[$ATTR_18]]()[%[[NUM_EL_TO_STORE]]]
// CHECK32: %[[NEW_MASK:.+]] = vector.create_mask %[[MASK_IDX]] : vector<1xi1>
// CHECK32: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<1xi32>
@@ -625,30 +726,32 @@ func.func @vector_maskedstore_i4(
// -----
-func.func @vector_maskedstore_i8_constant_mask(%arg0: index, %arg1: index, %value: vector<8xi8>) {
+// Note: emulating a maskedstore requires a static, container-aligned offset
+// (a dynamic offset makes it impossible to prove alignment and the pattern
+// bails out), hence the constant indices.
+func.func @vector_maskedstore_i8_constant_mask(%value: vector<8xi8>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
%0 = memref.alloc() : memref<3x8xi8>
%mask = vector.constant_mask [4] : vector<8xi1>
- vector.maskedstore %0[%arg0, %arg1], %mask, %value : memref<3x8xi8>, vector<8xi1>, vector<8xi8>
+ vector.maskedstore %0[%c1, %c0], %mask, %value : memref<3x8xi8>, vector<8xi1>, vector<8xi8>
return
}
// Expect no conversions, i8 is supported.
// CHECK: func @vector_maskedstore_i8_constant_mask(
-// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]
-// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]]
// CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]]
-// CHECK-NEXT: %[[ALLOC:.+]] = memref.alloc() : memref<3x8xi8>
-// CHECK-NEXT: %[[MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
-// CHECK-NEXT: vector.maskedstore %[[ALLOC]][%[[ARG0]], %[[ARG1]]], %[[MASK]], %[[VAL]]
+// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index
+// CHECK-DAG: %[[ALLOC:.+]] = memref.alloc() : memref<3x8xi8>
+// CHECK: %[[MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
+// CHECK-NEXT: vector.maskedstore %[[ALLOC]][%[[C1]], %[[C0]]], %[[MASK]], %[[VAL]]
// CHECK-NEXT: return
-// CHECK32-DAG: #[[LOAD_IDX_MAP:.+]] = affine_map<()[s0, s1] -> (s0 * 2 + s1 floordiv 4)>
// CHECK32: func @vector_maskedstore_i8_constant_mask(
-// CHECK32-SAME: %[[ARG0:[a-zA-Z0-9]+]]
-// CHECK32-SAME: %[[ARG1:[a-zA-Z0-9]+]]
// CHECK32-SAME: %[[VAL:[a-zA-Z0-9]+]]
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<6xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
-// CHECK32: %[[LIDX:.+]] = affine.apply #[[LOAD_IDX_MAP]]()[%[[ARG0]], %[[ARG1]]]
+// CHECK32: %[[LIDX:.+]] = arith.constant 2 : index
// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [1] : vector<2xi1>
// CHECK32: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<2xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LIDX]]], %[[NEW_MASK]], %[[PASS_THRU]]
@@ -659,26 +762,26 @@ func.func @vector_maskedstore_i8_constant_mask(%arg0: index, %arg1: index, %valu
// -----
+// Note: emulating a maskedstore requires a static, container-aligned offset
+// (a dynamic offset makes it impossible to prove alignment and the pattern
+// bails out), hence the constant indices.
func.func @vector_maskedstore_i4_constant_mask(
- %idx_1: index,
- %idx_2: index,
%val_to_store: vector<8xi4>) {
+ %c0 = arith.constant 0 : index
+ %c1 = arith.constant 1 : index
%0 = memref.alloc() : memref<3x8xi4>
%mask = vector.constant_mask [4] : vector<8xi1>
- vector.maskedstore %0[%idx_1, %idx_2], %mask, %val_to_store :
+ vector.maskedstore %0[%c1, %c0], %mask, %val_to_store :
memref<3x8xi4>, vector<8xi1>, vector<8xi4>
return
}
-// CHECK: #[[$ATTR_12:.+]] = affine_map<()[s0, s1] -> (s0 * 4 + s1 floordiv 2)>
// CHECK: func.func @vector_maskedstore_i4_constant_mask(
-// CHECK-SAME: %[[IDX_1:[a-zA-Z0-9]+]]: index,
-// CHECK-SAME: %[[IDX_2:[a-zA-Z0-9]+]]: index,
// CHECK-SAME: %[[VAL_TO_STORE:[a-zA-Z0-9]+]]: vector<8xi4>) {
// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<12xi8>
// CHECK: %[[ORIG_MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
-// CHECK: %[[LIDX:.+]] = affine.apply #[[$ATTR_12]]()[%[[IDX_1]], %[[IDX_2]]]
+// CHECK: %[[LIDX:.+]] = arith.constant 4 : index
// CHECK: %[[NEW_MASK:.+]] = vector.constant_mask [2] : vector<4xi1>
// CHECK: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<4xi8>
// CHECK: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LIDX]]], %[[NEW_MASK]], %[[PASS_THRU]] : memref<12xi8>, vector<4xi1>, vector<4xi8> into vector<4xi8>
@@ -687,14 +790,11 @@ func.func @vector_maskedstore_i4_constant_mask(
// CHECK: %[[NEW_VAL:.+]] = vector.bitcast %[[SELECT]] : vector<8xi4> to vector<4xi8>
// CHECK: vector.maskedstore %[[ALLOC]]{{\[}}%[[LIDX]]], %[[NEW_MASK]], %[[NEW_VAL]] : memref<12xi8>, vector<4xi1>, vector<4xi8>
-// CHECK32: #[[$ATTR_20:.+]] = affine_map<()[s0, s1] -> (s0 + s1 floordiv 8)>
// CHECK32: func.func @vector_maskedstore_i4_constant_mask(
-// CHECK32-SAME: %[[IDX_1:[a-zA-Z0-9]+]]: index,
-// CHECK32-SAME: %[[IDX_2:[a-zA-Z0-9]+]]: index,
// CHECK32-SAME: %[[VAL_TO_STORE:[a-zA-Z0-9]+]]: vector<8xi4>) {
// CHECK32: %[[ALLOC:.+]] = memref.alloc() : memref<3xi32>
// CHECK32: %[[ORIG_MASK:.+]] = vector.constant_mask [4] : vector<8xi1>
-// CHECK32: %[[LIDX:.+]] = affine.apply #[[$ATTR_20]]()[%[[IDX_1]], %[[IDX_2]]]
+// CHECK32: %[[LIDX:.+]] = arith.constant 1 : index
// CHECK32: %[[NEW_MASK:.+]] = vector.constant_mask [1] : vector<1xi1>
// CHECK32: %[[PASS_THRU:.+]] = arith.constant dense<0> : vector<1xi32>
// CHECK32: %[[LOAD:.+]] = vector.maskedload %[[ALLOC]][%[[LIDX]]], %[[NEW_MASK]], %[[PASS_THRU]] : memref<3xi32>, vector<1xi1>, vector<1xi32> into vector<1xi32>
More information about the Mlir-commits
mailing list