[llvm-branch-commits] [mlir] [mlir][vector] Do not fold contiguous gather/scatter on non-unit strides (PR #241601)
Zhijie Wang via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 15 10:31:53 PST 2026
https://github.com/aeft updated https://github.com/llvm/llvm-project/pull/241601
>From e0aa9e17f6147774d5b9545a5bbeda5c18eae3b0 Mon Sep 17 00:00:00 2001
From: Zhijie Wang <yesterda9 at gmail.com>
Date: Wed, 11 Nov 2026 15:58:11 -0800
Subject: [PATCH] [mlir][vector] Do not fold contiguous gather/scatter on
non-unit strides
Assisted-by: Claude Opus 5.5
---
mlir/lib/Dialect/Vector/IR/VectorOps.cpp | 14 ++++++-
mlir/test/Dialect/Vector/canonicalize.mlir | 44 ++++++++++++++++++++++
2 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 89f5e4f1559f70..9b39c121718cdc 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -6637,12 +6637,17 @@ class FoldContiguousGather final : public OpRewritePattern<GatherOp> {
using Base::Base;
LogicalResult matchAndRewrite(GatherOp op,
PatternRewriter &rewriter) const override {
- if (!isa<MemRefType>(op.getBase().getType()))
+ auto memRefTy = dyn_cast<MemRefType>(op.getBase().getType());
+ if (!memRefTy)
return rewriter.notifyMatchFailure(op, "base must be of memref type");
if (failed(isZeroBasedContiguousSeq(op.getIndices())))
return failure();
+ if (!isValidLoadStoreMemRefLayout(op.getVectorType(), memRefTy))
+ return rewriter.notifyMatchFailure(
+ op, "most minor memref dim must have unit stride");
+
rewriter.replaceOpWithNewOp<MaskedLoadOp>(op, op.getType(), op.getBase(),
op.getOffsets(), op.getMask(),
op.getPassThru());
@@ -6736,12 +6741,17 @@ class FoldContiguousScatter final : public OpRewritePattern<ScatterOp> {
PatternRewriter &rewriter) const override {
// Fold only for memrefs: the replacement uses maskedstore, which does not
// support tensor bases. Tensor cases intentionally bail out.
- if (!isa<MemRefType>(op.getBase().getType()))
+ auto memRefTy = dyn_cast<MemRefType>(op.getBase().getType());
+ if (!memRefTy)
return failure();
if (failed(isZeroBasedContiguousSeq(op.getIndices())))
return failure();
+ if (!isValidLoadStoreMemRefLayout(op.getVectorType(), memRefTy))
+ return rewriter.notifyMatchFailure(
+ op, "most minor memref dim must have unit stride");
+
rewriter.replaceOpWithNewOp<MaskedStoreOp>(
op, op.getBase(), op.getOffsets(), op.getMask(), op.getValueToStore());
return success();
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 3d0e510b8e1703..46c7e6db2e7dee 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -4462,6 +4462,36 @@ func.func @no_fold_contiguous_gather_tensor(%base: tensor<8xf32>, %mask: vector<
// -----
+// CHECK-LABEL: @no_fold_contiguous_gather_non_unit_stride
+// CHECK: vector.gather
+// CHECK-NOT: vector.maskedload
+func.func @no_fold_contiguous_gather_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+ %mask: vector<2xi1>, %pass_thru: vector<2xf32>) -> vector<2xf32> {
+ %c0 = arith.constant 0 : index
+ %indices = arith.constant dense<[0, 1]> : vector<2xindex>
+ %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru :
+ memref<4xf32, strided<[3]>>, vector<2xindex>, vector<2xi1>, vector<2xf32> into vector<2xf32>
+ return %0 : vector<2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @contiguous_gather_1elem_non_unit_stride
+// CHECK-SAME: (%[[BASE:.*]]: memref<4xf32, strided<[3]>>, %[[MASK:.*]]: vector<1xi1>, %[[PASSTHRU:.*]]: vector<1xf32>)
+// CHECK: %[[C0:.*]] = arith.constant 0 : index
+// CHECK: %[[R:.*]] = vector.maskedload %[[BASE]][%[[C0]]], %[[MASK]], %[[PASSTHRU]] : memref<4xf32, strided<[3]>>, vector<1xi1>, vector<1xf32> into vector<1xf32>
+// CHECK: return %[[R]]
+func.func @contiguous_gather_1elem_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+ %mask: vector<1xi1>, %pass_thru: vector<1xf32>) -> vector<1xf32> {
+ %c0 = arith.constant 0 : index
+ %indices = arith.constant dense<[0]> : vector<1xindex>
+ %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru :
+ memref<4xf32, strided<[3]>>, vector<1xindex>, vector<1xi1>, vector<1xf32> into vector<1xf32>
+ return %0 : vector<1xf32>
+}
+
+// -----
+
// CHECK-LABEL: @gather_broadcast(
// TODO: Broadcast is not supported yet
// CHECK: %[[R:.*]] = vector.gather
@@ -4540,6 +4570,20 @@ func.func @no_fold_contiguous_scatter_tensor(%base: tensor<16xf32>,
// -----
+// CHECK-LABEL: @no_fold_contiguous_scatter_non_unit_stride
+// CHECK: vector.scatter
+// CHECK-NOT: vector.maskedstore
+func.func @no_fold_contiguous_scatter_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+ %mask: vector<2xi1>, %value: vector<2xf32>) {
+ %c0 = arith.constant 0 : index
+ %indices = vector.step : vector<2xindex>
+ vector.scatter %base[%c0][%indices], %mask, %value :
+ memref<4xf32, strided<[3]>>, vector<2xindex>, vector<2xi1>, vector<2xf32>
+ return
+}
+
+// -----
+
// CHECK-LABEL: @scatter_memref_all_false
// CHECK-SAME: (%[[BASE:.*]]: memref<?xf32>, %[[INDEX:.*]]: vector<16xindex>, %[[VALUE:.*]]: vector<16xf32>)
// CHECK-NEXT: return
More information about the llvm-branch-commits
mailing list