[llvm-branch-commits] [mlir] [mlir][vector] Do not fold contiguous gather/scatter on non-unit strides (PR #241601)

Zhijie Wang via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 15 10:31:53 PST 2026


https://github.com/aeft updated https://github.com/llvm/llvm-project/pull/241601

>From e0aa9e17f6147774d5b9545a5bbeda5c18eae3b0 Mon Sep 17 00:00:00 2001
From: Zhijie Wang <yesterda9 at gmail.com>
Date: Wed, 11 Nov 2026 15:58:11 -0800
Subject: [PATCH] [mlir][vector] Do not fold contiguous gather/scatter on
 non-unit strides

Assisted-by: Claude Opus 5.5
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp   | 14 ++++++-
 mlir/test/Dialect/Vector/canonicalize.mlir | 44 ++++++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 89f5e4f1559f70..9b39c121718cdc 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -6637,12 +6637,17 @@ class FoldContiguousGather final : public OpRewritePattern<GatherOp> {
   using Base::Base;
   LogicalResult matchAndRewrite(GatherOp op,
                                 PatternRewriter &rewriter) const override {
-    if (!isa<MemRefType>(op.getBase().getType()))
+    auto memRefTy = dyn_cast<MemRefType>(op.getBase().getType());
+    if (!memRefTy)
       return rewriter.notifyMatchFailure(op, "base must be of memref type");
 
     if (failed(isZeroBasedContiguousSeq(op.getIndices())))
       return failure();
 
+    if (!isValidLoadStoreMemRefLayout(op.getVectorType(), memRefTy))
+      return rewriter.notifyMatchFailure(
+          op, "most minor memref dim must have unit stride");
+
     rewriter.replaceOpWithNewOp<MaskedLoadOp>(op, op.getType(), op.getBase(),
                                               op.getOffsets(), op.getMask(),
                                               op.getPassThru());
@@ -6736,12 +6741,17 @@ class FoldContiguousScatter final : public OpRewritePattern<ScatterOp> {
                                 PatternRewriter &rewriter) const override {
     // Fold only for memrefs: the replacement uses maskedstore, which does not
     // support tensor bases. Tensor cases intentionally bail out.
-    if (!isa<MemRefType>(op.getBase().getType()))
+    auto memRefTy = dyn_cast<MemRefType>(op.getBase().getType());
+    if (!memRefTy)
       return failure();
 
     if (failed(isZeroBasedContiguousSeq(op.getIndices())))
       return failure();
 
+    if (!isValidLoadStoreMemRefLayout(op.getVectorType(), memRefTy))
+      return rewriter.notifyMatchFailure(
+          op, "most minor memref dim must have unit stride");
+
     rewriter.replaceOpWithNewOp<MaskedStoreOp>(
         op, op.getBase(), op.getOffsets(), op.getMask(), op.getValueToStore());
     return success();
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 3d0e510b8e1703..46c7e6db2e7dee 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -4462,6 +4462,36 @@ func.func @no_fold_contiguous_gather_tensor(%base: tensor<8xf32>, %mask: vector<
 
 // -----
 
+// CHECK-LABEL: @no_fold_contiguous_gather_non_unit_stride
+//       CHECK:   vector.gather
+//   CHECK-NOT:   vector.maskedload
+func.func @no_fold_contiguous_gather_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+    %mask: vector<2xi1>, %pass_thru: vector<2xf32>) -> vector<2xf32> {
+  %c0 = arith.constant 0 : index
+  %indices = arith.constant dense<[0, 1]> : vector<2xindex>
+  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru :
+    memref<4xf32, strided<[3]>>, vector<2xindex>, vector<2xi1>, vector<2xf32> into vector<2xf32>
+  return %0 : vector<2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @contiguous_gather_1elem_non_unit_stride
+//  CHECK-SAME:   (%[[BASE:.*]]: memref<4xf32, strided<[3]>>, %[[MASK:.*]]: vector<1xi1>, %[[PASSTHRU:.*]]: vector<1xf32>)
+//       CHECK:   %[[C0:.*]] = arith.constant 0 : index
+//       CHECK:   %[[R:.*]] = vector.maskedload %[[BASE]][%[[C0]]], %[[MASK]], %[[PASSTHRU]] : memref<4xf32, strided<[3]>>, vector<1xi1>, vector<1xf32> into vector<1xf32>
+//       CHECK:   return %[[R]]
+func.func @contiguous_gather_1elem_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+    %mask: vector<1xi1>, %pass_thru: vector<1xf32>) -> vector<1xf32> {
+  %c0 = arith.constant 0 : index
+  %indices = arith.constant dense<[0]> : vector<1xindex>
+  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru :
+    memref<4xf32, strided<[3]>>, vector<1xindex>, vector<1xi1>, vector<1xf32> into vector<1xf32>
+  return %0 : vector<1xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @gather_broadcast(
 // TODO: Broadcast is not supported yet
 //       CHECK:   %[[R:.*]] = vector.gather
@@ -4540,6 +4570,20 @@ func.func @no_fold_contiguous_scatter_tensor(%base: tensor<16xf32>,
 
 // -----
 
+// CHECK-LABEL: @no_fold_contiguous_scatter_non_unit_stride
+//       CHECK:   vector.scatter
+//   CHECK-NOT:   vector.maskedstore
+func.func @no_fold_contiguous_scatter_non_unit_stride(%base: memref<4xf32, strided<[3]>>,
+    %mask: vector<2xi1>, %value: vector<2xf32>) {
+  %c0 = arith.constant 0 : index
+  %indices = vector.step : vector<2xindex>
+  vector.scatter %base[%c0][%indices], %mask, %value :
+    memref<4xf32, strided<[3]>>, vector<2xindex>, vector<2xi1>, vector<2xf32>
+  return
+}
+
+// -----
+
 // CHECK-LABEL: @scatter_memref_all_false
 //  CHECK-SAME:   (%[[BASE:.*]]: memref<?xf32>, %[[INDEX:.*]]: vector<16xindex>, %[[VALUE:.*]]: vector<16xf32>)
 //  CHECK-NEXT:   return



More information about the llvm-branch-commits mailing list