[Mlir-commits] [mlir] [mlir][VectorToLLVM] add opt-in `enable-gep-inbounds-nuw` pass flag for `vector.load/store` (PR #202118)
Federico Bruzzone
llvmlistbot at llvm.org
Wed Jun 17 23:37:55 PDT 2026
https://github.com/FedericoBruzzone updated https://github.com/llvm/llvm-project/pull/202118
>From f174b30074d92c8c11ce527dfa3fd68ea20a9f7b Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Sun, 7 Jun 2026 12:52:36 +0200
Subject: [PATCH 1/5] [mlir][VectorToLLVM] emit inbounds|nuw GEP flags when
lowering vector.load/store
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 13 +++-
.../Conversion/GPUCommon/transfer_write.mlir | 2 +-
.../VectorToLLVM/vector-scalable-memcpy.mlir | 4 +-
.../vector-to-llvm-interface.mlir | 60 +++++++++----------
.../VectorToLLVM/vector-xfer-to-llvm.mlir | 4 +-
.../SuperVectorize/vectorize_2d_inbounds.mlir | 5 +-
.../vectorize_inbounds_llvmopt.mlir | 49 +++++++++++++++
7 files changed, 99 insertions(+), 38 deletions(-)
create mode 100644 mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 43e0824fef6cd..0ba48b1f3862e 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -256,10 +256,19 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
"could not resolve alignment");
// Resolve address.
+ // Per vector.load/store spec, indices must be in-bounds (0 <= idx <
+ // dim_size). Emit inbounds|nuw so LLVM can apply no-wrap optimizations on
+ // the generated index arithmetic and GEP. Masked variants are designed for
+ // near-boundary access, so they conservatively omit these flags.
+ LLVM::GEPNoWrapFlags noWrapFlags = LLVM::GEPNoWrapFlags::none;
+ if constexpr (std::is_same_v<LoadOrStoreOp, vector::LoadOp> ||
+ std::is_same_v<LoadOrStoreOp, vector::StoreOp>)
+ noWrapFlags = LLVM::GEPNoWrapFlags::inbounds | LLVM::GEPNoWrapFlags::nuw;
auto vtype = cast<VectorType>(
this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
- Value dataPtr = this->getStridedElementPtr(
- rewriter, loc, memRefTy, adaptor.getBase(), adaptor.getIndices());
+ Value dataPtr =
+ this->getStridedElementPtr(rewriter, loc, memRefTy, adaptor.getBase(),
+ adaptor.getIndices(), noWrapFlags);
replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, dataPtr, align,
rewriter);
return success();
diff --git a/mlir/test/Conversion/GPUCommon/transfer_write.mlir b/mlir/test/Conversion/GPUCommon/transfer_write.mlir
index 4d2ae8c39240c..34dd28077eb47 100644
--- a/mlir/test/Conversion/GPUCommon/transfer_write.mlir
+++ b/mlir/test/Conversion/GPUCommon/transfer_write.mlir
@@ -3,7 +3,7 @@
// CHECK-LABEL: @warp_extract
// CHECK-SAME: %[[VEC:[a-zA-Z0-9_]+]]: vector<1xf32>
// CHECK:%[[BASE:[0-9]+]] = llvm.extractvalue
-// CHECK:%[[PTR:[0-9]+]] = llvm.getelementptr %[[BASE]]
+// CHECK:%[[PTR:[0-9]+]] = llvm.getelementptr inbounds|nuw %[[BASE]]
// CHECK:llvm.store %[[VEC]], %[[PTR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
func.func @warp_extract(%arg0: index, %arg1: memref<1024x1024xf32>, %arg2: vector<1xf32>) {
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
index 80e6caa05db5e..58fb69e03c85b 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
@@ -12,11 +12,11 @@ func.func @vector_scalable_memcopy(%src : memref<?xf32>, %dst : memref<?xf32>, %
scf.for %i0 = %c0 to %size step %step {
// CHECK: [[DATAIDX:%[0-9]+]] = builtin.unrealized_conversion_cast [[LOOPIDX]] : index to i64
// CHECK: [[SRCMEM:%[0-9]+]] = llvm.extractvalue [[SRCMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: [[LDVAL:%[0-9]+]] = llvm.load [[SRCPTR]]{{.*}}: !llvm.ptr -> vector<[4]xf32>
%0 = vector.load %src[%i0] : memref<?xf32>, vector<[4]xf32>
// CHECK: [[DSTMEM:%[0-9]+]] = llvm.extractvalue [[DSTMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: llvm.store [[LDVAL]], [[DSTPTR]]{{.*}}: vector<[4]xf32>, !llvm.ptr
vector.store %0, %dst[%i0] : memref<?xf32>, vector<[4]xf32>
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index d570d46e11b4a..0dda2333f962f 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1588,9 +1588,9 @@ func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector
// CHECK-LABEL: func @load
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1602,9 +1602,9 @@ func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @load_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1616,9 +1616,9 @@ func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : inde
// CHECK-LABEL: func @load_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1630,9 +1630,9 @@ func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @load_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1670,9 +1670,9 @@ func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vec
// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
// CHECK: return %[[RES]] : vector<f32>
@@ -1701,9 +1701,9 @@ func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK-LABEL: func @store
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
// -----
@@ -1716,9 +1716,9 @@ func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @store_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1731,9 +1731,9 @@ func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : ind
// CHECK-LABEL: func @store_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<4xf32>, !llvm.ptr
// -----
@@ -1746,9 +1746,9 @@ func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @store_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1787,9 +1787,9 @@ func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
// CHECK: return
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
index 18deadd0d7a79..1d998e09212b4 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
@@ -298,7 +298,7 @@ func.func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
@@ -314,7 +314,7 @@ func.func @transfer_read_1d_inbounds_scalable(%A : memref<?xf32>, %base: index)
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
index d95c5bdee79d7..10a96654c2b36 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
@@ -22,8 +22,11 @@
// CHECK-NOT: vector.transfer_write
// LLVM-LABEL: llvm.func @copy
-// Verify that in_bounds lowers to plain llvm.load/store, not masked intrinsics.
+// Verify that in_bounds lowers to plain llvm.load/store with inbounds|nuw GEP,
+// not masked intrinsics, enabling LLVM to apply no-wrap optimizations.
+// LLVM: llvm.getelementptr inbounds|nuw {{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// LLVM: llvm.load {{.*}} : !llvm.ptr -> vector<4xf32>
+// LLVM: llvm.getelementptr inbounds|nuw {{.*}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// LLVM: llvm.store {{.*}} : vector<4xf32>, !llvm.ptr
// LLVM-NOT: llvm.intr.masked.load
// LLVM-NOT: llvm.intr.masked.store
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
new file mode 100644
index 0000000000000..aa79ae373be79
--- /dev/null
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
@@ -0,0 +1,49 @@
+// Verify that the inbounds|nuw flags emitted on vector.load/store GEPs (when
+// in_bounds=true) enable LLVM to perform alias analysis and produce
+// well-optimized LLVM IR. The pipeline lowers MLIR all the way to native LLVM
+// IR via mlir-translate, then passes the result through LLVM's -O2 pipeline.
+//
+// RUN: mlir-opt %s \
+// RUN: --affine-super-vectorize="virtual-vector-size=4" \
+// RUN: --lower-affine \
+// RUN: --convert-scf-to-cf \
+// RUN: --expand-strided-metadata \
+// RUN: --convert-arith-to-llvm \
+// RUN: --convert-cf-to-llvm \
+// RUN: --convert-vector-to-llvm \
+// RUN: --finalize-memref-to-llvm \
+// RUN: --convert-func-to-llvm \
+// RUN: --reconcile-unrealized-casts \
+// RUN: | mlir-translate --mlir-to-llvmir \
+// RUN: | opt -S -passes="default<O2>" \
+// RUN: | FileCheck %s --check-prefix=OPT
+
+// OPT-LABEL: define void @copy(
+// After -O2, LLVM alias-analysis annotates the source arg as read-only and
+// the destination arg as write-only. This requires the GEP to carry inbounds
+// and nuw flags (produced by our vector.load/store lowering fix) so LLVM can
+// prove the two memory regions do not overlap.
+// OPT-SAME: readonly
+// OPT-SAME: writeonly
+
+// The GEP for the load carries inbounds nuw — our fix propagated through opt.
+// OPT: getelementptr inbounds nuw
+// The MLIR-level vectorization (vector<4xf32> from affine-super-vectorize)
+// must be preserved through LLVM optimisation — no scalar regression.
+// OPT-NEXT: load <4 x float>
+// OPT-NEXT: getelementptr inbounds nuw
+// OPT-NEXT: store <4 x float>
+
+// No masked-load/store intrinsics: in_bounds=true correctly skipped masking.
+// OPT-NOT: @llvm.masked.load
+// OPT-NOT: @llvm.masked.store
+
+func.func @copy(%A: memref<512x512xf32>, %B: memref<512x512xf32>) {
+ affine.for %i = 0 to 512 {
+ affine.for %j = 0 to 512 {
+ %v = affine.load %A[%i, %j] : memref<512x512xf32>
+ affine.store %v, %B[%i, %j] : memref<512x512xf32>
+ }
+ }
+ return
+}
>From 0822bb002b144cb200fb3ec62a7f3746b17cd6fc Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Sun, 14 Jun 2026 14:57:46 +0200
Subject: [PATCH 2/5] [mlir][vector] Add opt-in `inbounds`/`nneg` flags to
`vector.load`/`vector.store`
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
.../mlir/Dialect/Vector/IR/VectorOps.td | 65 +++++++-
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 30 +++-
.../Vector/Transforms/LowerVectorGather.cpp | 6 +-
.../Vector/Transforms/LowerVectorTransfer.cpp | 17 ++-
.../VectorToLLVM/vector-scalable-memcpy.mlir | 4 +-
.../vector-to-llvm-interface.mlir | 141 ++++++++++++++----
.../vectorize_inbounds_llvmopt.mlir | 49 ------
.../vector-transfer-to-vector-load-store.mlir | 30 ++--
8 files changed, 227 insertions(+), 115 deletions(-)
delete mode 100644 mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 5acf2b4ab7649..a98a14ae54d29 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1730,32 +1730,58 @@ def Vector_LoadOp : Vector_Op<"load", [
load operation. It must be a positive power of 2. The operation must access
memory at an address aligned to this boundary. Violating this requirement
triggers immediate undefined behavior.
+
+ Out-of-bounds behavior is implementation-defined at the `vector` level: both
+ out-of-bounds start indices and the out-of-bounds tail of an in-bounds start
+ (e.g. `%memref[%c0] : memref<7xf32>, vector<8xf32>`) have target-specific
+ behavior (e.g. SPIR-V robust buffer access or AMDGPU buffer fat pointers may
+ define them to read zeros). The optional `inbounds` and `nneg` unit
+ attributes let a producer opt in to stronger assumptions used by lowerings:
+ * `inbounds` asserts the access is in-bounds, i.e. `0 <= idx < dim_size`
+ for every dimension. When set, the LLVM lowering may emit
+ `llvm.getelementptr inbounds`.
+ * `nneg` asserts that every index is non-negative. When set (and the
+ memref has non-negative strides), the LLVM lowering may additionally
+ emit the `nuw` flag.
+
+ Both default to unset, in which case no flag is emitted and the
+ target-defined out-of-bounds behavior above is preserved. Setting either
+ attribute when it does not hold is undefined behavior under the LLVM
+ lowering.
}];
let arguments = (ins Arg<AnyMemRef, "the reference to load from",
[MemRead]>:$base,
Variadic<Index>:$indices,
DefaultValuedOptionalAttr<BoolAttr, "false">:$nontemporal,
- OptionalAttr<IntValidAlignment<I64Attr>>: $alignment);
+ OptionalAttr<IntValidAlignment<I64Attr>>: $alignment,
+ UnitAttr:$inbounds,
+ UnitAttr:$nneg);
let builders = [
OpBuilder<(ins "VectorType":$resultType,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
+ CArg<"bool", "false">:$inbounds,
+ CArg<"bool", "false">:$nneg), [{
return build($_builder, $_state, resultType, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr);
+ nullptr,
+ inbounds, nneg);
}]>,
OpBuilder<(ins "TypeRange":$resultTypes,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
+ CArg<"bool", "false">:$inbounds,
+ CArg<"bool", "false">:$nneg), [{
return build($_builder, $_state, resultTypes, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr);
+ nullptr,
+ inbounds, nneg);
}]>
];
@@ -1847,6 +1873,24 @@ def Vector_StoreOp : Vector_Op<"store", [
store operation. It must be a positive power of 2. The operation must access
memory at an address aligned to this boundary. Violating this requirement
triggers immediate undefined behavior.
+
+ Out-of-bounds behavior is implementation-defined at the `vector` level: both
+ out-of-bounds start indices and the out-of-bounds tail of an in-bounds start
+ (e.g. `%memref[%c0] : memref<7xf32>, vector<8xf32>`) have target-specific
+ behavior (e.g. SPIR-V robust buffer access or AMDGPU buffer fat pointers may
+ define them to drop the write). The optional `inbounds` and `nneg` unit
+ attributes let a producer opt in to stronger assumptions used by lowerings:
+ * `inbounds` asserts the access is in-bounds, i.e. `0 <= idx < dim_size`
+ for every dimension. When set, the LLVM lowering may emit
+ `llvm.getelementptr inbounds`.
+ * `nneg` asserts that every index is non-negative. When set (and the
+ memref has non-negative strides), the LLVM lowering may additionally
+ emit the `nuw` flag.
+
+ Both default to unset, in which case no flag is emitted and the
+ target-defined out-of-bounds behavior above is preserved. Setting either
+ attribute when it does not hold is undefined behavior under the LLVM
+ lowering.
}];
let arguments = (ins
@@ -1855,17 +1899,22 @@ def Vector_StoreOp : Vector_Op<"store", [
[MemWrite]>:$base,
Variadic<Index>:$indices,
DefaultValuedOptionalAttr<BoolAttr, "false">:$nontemporal,
- OptionalAttr<IntValidAlignment<I64Attr>>: $alignment);
+ OptionalAttr<IntValidAlignment<I64Attr>>: $alignment,
+ UnitAttr:$inbounds,
+ UnitAttr:$nneg);
let builders = [
OpBuilder<(ins "Value":$valueToStore,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
+ CArg<"bool", "false">:$inbounds,
+ CArg<"bool", "false">:$nneg), [{
return build($_builder, $_state, valueToStore, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr);
+ nullptr,
+ inbounds, nneg);
}]>
];
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 0ba48b1f3862e..33d2d0bc8f055 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -223,6 +223,16 @@ static void replaceLoadOrStoreOp(vector::MaskedStoreOp storeOp,
storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align);
}
+/// Returns true if all strides of `memRefTy` are static and non-negative. A
+/// negative (or dynamic, hence unknown-sign) stride would make `mul nuw` on the
+/// index arithmetic wrap, so `nuw` must not be emitted in that case.
+static bool hasNonNegativeStrides(MemRefType memRefTy) {
+ auto [strides, offset] = memRefTy.getStridesAndOffset();
+ return llvm::all_of(strides, [](int64_t stride) {
+ return !ShapedType::isDynamic(stride) && stride >= 0;
+ });
+}
+
/// Conversion pattern for a vector.load, vector.store, vector.maskedload, and
/// vector.maskedstore.
template <class LoadOrStoreOp>
@@ -256,14 +266,22 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
"could not resolve alignment");
// Resolve address.
- // Per vector.load/store spec, indices must be in-bounds (0 <= idx <
- // dim_size). Emit inbounds|nuw so LLVM can apply no-wrap optimizations on
- // the generated index arithmetic and GEP. Masked variants are designed for
- // near-boundary access, so they conservatively omit these flags.
+ // `vector.load`/`vector.store` may carry `inbounds`/`nneg` assertions about
+ // their indices (see the op docs). Translate them into GEP no-wrap flags so
+ // LLVM can apply no-wrap optimizations on the generated index arithmetic
+ // and GEP. When the attributes are absent (and for the masked variants,
+ // which target near-boundary access) no flag is emitted, preserving any
+ // target-defined out-of-bounds behavior.
LLVM::GEPNoWrapFlags noWrapFlags = LLVM::GEPNoWrapFlags::none;
if constexpr (std::is_same_v<LoadOrStoreOp, vector::LoadOp> ||
- std::is_same_v<LoadOrStoreOp, vector::StoreOp>)
- noWrapFlags = LLVM::GEPNoWrapFlags::inbounds | LLVM::GEPNoWrapFlags::nuw;
+ std::is_same_v<LoadOrStoreOp, vector::StoreOp>) {
+ if (loadOrStoreOp.getInbounds())
+ noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::inbounds;
+ // `nuw` additionally requires the whole offset computation to be
+ // non-negative: non-negative indices (nneg) *and* non-negative strides.
+ if (loadOrStoreOp.getNneg() && hasNonNegativeStrides(memRefTy))
+ noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::nuw;
+ }
auto vtype = cast<VectorType>(
this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
Value dataPtr =
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 5a8e473d39360..111074dff9191 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -293,9 +293,9 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
if (isa<MemRefType>(base.getType())) {
// `vector.load` does not support scalar result; emit a vector load
// and extract the single result instead.
- Value load =
- vector::LoadOp::create(b, loc, elemVecTy, base, loadOffsets,
- nontemporalAttr, alignmentAttr);
+ Value load = vector::LoadOp::create(
+ b, loc, elemVecTy, base, loadOffsets, nontemporalAttr,
+ alignmentAttr, /*inbounds=*/nullptr, /*nneg=*/nullptr);
int64_t zeroIdx[1] = {0};
extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
} else {
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 2cf8f0beaa4de..77f54f639efad 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -474,9 +474,15 @@ struct TransferReadToVectorLoadLowering
rewriter, read.getLoc(), unbroadcastedVectorType, read.getBase(),
read.getIndices(), read.getMask(), fill);
} else {
+ // This pattern only matches when no dimension is out-of-bounds (see the
+ // `hasOutOfBoundsDim` check above), so the access is in-bounds and the
+ // indices are non-negative. Propagate that as `inbounds`/`nneg` so the
+ // LLVM lowering can emit the corresponding GEP no-wrap flags.
res = vector::LoadOp::create(rewriter, read.getLoc(),
unbroadcastedVectorType, read.getBase(),
- read.getIndices());
+ read.getIndices(), /*nontemporal=*/false,
+ /*alignment=*/llvm::MaybeAlign(),
+ /*inbounds=*/true, /*nneg=*/true);
}
// Insert a broadcasting op if required.
@@ -570,8 +576,15 @@ struct TransferWriteToVectorStoreLowering
write.getIndices(), write.getMask(),
write.getVector());
} else {
+ // This pattern only matches when no dimension is out-of-bounds (see the
+ // `hasOutOfBoundsDim` check above), so the access is in-bounds and the
+ // indices are non-negative. Propagate that as `inbounds`/`nneg` so the
+ // LLVM lowering can emit the corresponding GEP no-wrap flags.
vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
- write.getBase(), write.getIndices());
+ write.getBase(), write.getIndices(),
+ /*nontemporal=*/false,
+ /*alignment=*/llvm::MaybeAlign(),
+ /*inbounds=*/true, /*nneg=*/true);
}
// There's no return value for StoreOps. Use Value() to signal success to
// matchAndRewrite.
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
index 58fb69e03c85b..80e6caa05db5e 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
@@ -12,11 +12,11 @@ func.func @vector_scalable_memcopy(%src : memref<?xf32>, %dst : memref<?xf32>, %
scf.for %i0 = %c0 to %size step %step {
// CHECK: [[DATAIDX:%[0-9]+]] = builtin.unrealized_conversion_cast [[LOOPIDX]] : index to i64
// CHECK: [[SRCMEM:%[0-9]+]] = llvm.extractvalue [[SRCMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: [[LDVAL:%[0-9]+]] = llvm.load [[SRCPTR]]{{.*}}: !llvm.ptr -> vector<[4]xf32>
%0 = vector.load %src[%i0] : memref<?xf32>, vector<[4]xf32>
// CHECK: [[DSTMEM:%[0-9]+]] = llvm.extractvalue [[DSTMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: llvm.store [[LDVAL]], [[DSTPTR]]{{.*}}: vector<[4]xf32>, !llvm.ptr
vector.store %0, %dst[%i0] : memref<?xf32>, vector<[4]xf32>
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 0dda2333f962f..63f9e7e3a4016 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1588,9 +1588,9 @@ func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector
// CHECK-LABEL: func @load
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1602,9 +1602,9 @@ func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @load_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1616,9 +1616,9 @@ func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : inde
// CHECK-LABEL: func @load_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1630,9 +1630,9 @@ func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @load_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1670,9 +1670,9 @@ func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vec
// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
// CHECK: return %[[RES]] : vector<f32>
@@ -1701,9 +1701,9 @@ func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK-LABEL: func @store
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
// -----
@@ -1716,9 +1716,9 @@ func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @store_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1731,9 +1731,9 @@ func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : ind
// CHECK-LABEL: func @store_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<4xf32>, !llvm.ptr
// -----
@@ -1746,9 +1746,9 @@ func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @store_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1787,9 +1787,9 @@ func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
// CHECK: return
@@ -1805,6 +1805,87 @@ func.func @store_with_alignment(%memref : memref<200x100xf32>, %i : index, %j :
// -----
+//===----------------------------------------------------------------------===//
+// vector.load / vector.store inbounds / nneg flags
+//===----------------------------------------------------------------------===//
+
+// `inbounds` alone lowers to `getelementptr inbounds` (which implies nusw, hence
+// `nsw` on the index arithmetic). No `nuw` without `nneg`.
+func.func @load_inbounds(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] {inbounds} : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_inbounds
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+// `inbounds` + `nneg` on a non-negative-strided memref lowers to
+// `getelementptr inbounds|nuw` and `overflow<nsw, nuw>`.
+func.func @load_inbounds_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] {inbounds, nneg} : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_inbounds_nneg
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+// `nneg` alone (no `inbounds`) lowers to `getelementptr nuw` / `overflow<nuw>`.
+// This is the out-of-bounds-but-non-negative case (e.g. AMDGPU buffer fat
+// pointers), where we must not claim `inbounds`.
+func.func @load_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] {nneg} : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_nneg
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+// `nneg` must not produce `nuw` when a stride is negative: `mul nuw idx, stride`
+// would wrap. The negative major stride here disables `nuw` (and `inbounds` is
+// not requested), so a plain GEP is emitted.
+func.func @load_nneg_negative_stride(%memref : memref<200x100xf32, strided<[-100, 1], offset: ?>>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] {nneg} : memref<200x100xf32, strided<[-100, 1], offset: ?>>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_nneg_negative_stride
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+// Store side: `inbounds` + `nneg` lowers to `getelementptr inbounds|nuw`.
+func.func @store_inbounds_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<4xf32>
+ vector.store %val, %memref[%i, %j] {inbounds, nneg} : memref<200x100xf32>, vector<4xf32>
+ return
+}
+
+// CHECK-LABEL: func @store_inbounds_nneg
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
+
+// -----
+
//===----------------------------------------------------------------------===//
// vector.maskedload
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
deleted file mode 100644
index aa79ae373be79..0000000000000
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir
+++ /dev/null
@@ -1,49 +0,0 @@
-// Verify that the inbounds|nuw flags emitted on vector.load/store GEPs (when
-// in_bounds=true) enable LLVM to perform alias analysis and produce
-// well-optimized LLVM IR. The pipeline lowers MLIR all the way to native LLVM
-// IR via mlir-translate, then passes the result through LLVM's -O2 pipeline.
-//
-// RUN: mlir-opt %s \
-// RUN: --affine-super-vectorize="virtual-vector-size=4" \
-// RUN: --lower-affine \
-// RUN: --convert-scf-to-cf \
-// RUN: --expand-strided-metadata \
-// RUN: --convert-arith-to-llvm \
-// RUN: --convert-cf-to-llvm \
-// RUN: --convert-vector-to-llvm \
-// RUN: --finalize-memref-to-llvm \
-// RUN: --convert-func-to-llvm \
-// RUN: --reconcile-unrealized-casts \
-// RUN: | mlir-translate --mlir-to-llvmir \
-// RUN: | opt -S -passes="default<O2>" \
-// RUN: | FileCheck %s --check-prefix=OPT
-
-// OPT-LABEL: define void @copy(
-// After -O2, LLVM alias-analysis annotates the source arg as read-only and
-// the destination arg as write-only. This requires the GEP to carry inbounds
-// and nuw flags (produced by our vector.load/store lowering fix) so LLVM can
-// prove the two memory regions do not overlap.
-// OPT-SAME: readonly
-// OPT-SAME: writeonly
-
-// The GEP for the load carries inbounds nuw — our fix propagated through opt.
-// OPT: getelementptr inbounds nuw
-// The MLIR-level vectorization (vector<4xf32> from affine-super-vectorize)
-// must be preserved through LLVM optimisation — no scalar regression.
-// OPT-NEXT: load <4 x float>
-// OPT-NEXT: getelementptr inbounds nuw
-// OPT-NEXT: store <4 x float>
-
-// No masked-load/store intrinsics: in_bounds=true correctly skipped masking.
-// OPT-NOT: @llvm.masked.load
-// OPT-NOT: @llvm.masked.store
-
-func.func @copy(%A: memref<512x512xf32>, %B: memref<512x512xf32>) {
- affine.for %i = 0 to 512 {
- affine.for %j = 0 to 512 {
- %v = affine.load %A[%i, %j] : memref<512x512xf32>
- affine.store %v, %B[%i, %j] : memref<512x512xf32>
- }
- }
- return
-}
diff --git a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
index 8206c1a3ec865..10b01ead1e717 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
@@ -6,10 +6,10 @@
func.func @vector_transfer_ops_0d_memref(%mem: memref<f32>, %vec: vector<f32>) {
%f0 = arith.constant 0.0 : f32
-// CHECK-NEXT: %[[S:.*]] = vector.load %[[MEM]][] : memref<f32>, vector<f32>
+// CHECK-NEXT: %[[S:.*]] = vector.load %[[MEM]][] {inbounds, nneg} : memref<f32>, vector<f32>
%0 = vector.transfer_read %mem[], %f0 : memref<f32>, vector<f32>
-// CHECK-NEXT: vector.store %[[S]], %[[MEM]][] : memref<f32>, vector<f32>
+// CHECK-NEXT: vector.store %[[S]], %[[MEM]][] {inbounds, nneg} : memref<f32>, vector<f32>
vector.transfer_write %0, %mem[] : vector<f32>, memref<f32>
// CHECK-NEXT: vector.store %[[VEC]], %[[MEM]][] : memref<f32>, vector<f32>
@@ -36,8 +36,8 @@ func.func @vector_transfer_ops_0d_tensor(%src: tensor<f32>) -> vector<1xf32> {
// CHECK-LABEL: func @transfer_to_load(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -69,8 +69,8 @@ func.func @masked_transfer_to_load(%mem : memref<8x8xf32>, %idx : index, %mask :
// CHECK-LABEL: func @transfer_2D(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<2x4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<2x4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<2x4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<2x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<2x4xf32>
// CHECK-NEXT: }
@@ -85,8 +85,8 @@ func.func @transfer_2D(%mem : memref<8x8xf32>, %idx : index) -> vector<2x4xf32>
// CHECK-LABEL: func @transfer_vector_element(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xvector<2x4xf32>>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<2x4xf32>
// CHECK-NEXT: }
@@ -154,8 +154,8 @@ func.func @transfer_not_inbounds(%mem : memref<8x8xf32>, %idx : index) -> vector
// CHECK-LABEL: func @transfer_nondefault_layout(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32, #{{.*}}>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32, #{{.*}}>, vector<4xf32>
-// CHECK-NEXT: vector.store %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32, #{{.*}}>, vector<4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
+// CHECK-NEXT: vector.store %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -188,7 +188,7 @@ func.func @transfer_perm_map(%mem : memref<8x8xf32>, %idx : index) -> vector<4xf
// CHECK-LABEL: func @transfer_broadcasting(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<1xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<1xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<1xf32> to vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -205,7 +205,7 @@ func.func @transfer_broadcasting(%mem : memref<8x8xf32>, %idx : index) -> vector
// CHECK-LABEL: func @transfer_scalar(
// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<1xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<?x?xf32>, vector<1xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<?x?xf32>, vector<1xf32>
// CHECK-NEXT: return %[[RES]] : vector<1xf32>
// CHECK-NEXT: }
func.func @transfer_scalar(%mem : memref<?x?xf32>, %idx : index) -> vector<1xf32> {
@@ -218,7 +218,7 @@ func.func @transfer_scalar(%mem : memref<?x?xf32>, %idx : index) -> vector<1xf32
// CHECK-LABEL: func @transfer_broadcasting_2D(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4x4xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<1x1xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<1x1xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<1x1xf32> to vector<4x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4x4xf32>
// CHECK-NEXT: }
@@ -236,7 +236,7 @@ func.func @transfer_broadcasting_2D(%mem : memref<8x8xf32>, %idx : index) -> vec
// CHECK-LABEL: func @transfer_broadcasting_complex(
// CHECK-SAME: %[[MEM:.*]]: memref<10x20x30x8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<3x2x4x5xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]] : memref<10x20x30x8x8xf32>, vector<3x1x1x5xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<10x20x30x8x8xf32>, vector<3x1x1x5xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<3x1x1x5xf32> to vector<3x2x4x5xf32>
// CHECK-NEXT: return %[[RES]] : vector<3x2x4x5xf32>
// CHECK-NEXT: }
@@ -318,7 +318,7 @@ func.func @transfer_read_permutations(%mem_0 : memref<?x?xf32>, %mem_1 : memref<
// CHECK: vector.transpose %{{.*}}, [2, 1, 3, 0] : vector<16x14x7x8xf32> to vector<7x14x8x16xf32>
%6 = vector.transfer_read %mem_0[%c0, %c0], %cst {in_bounds = [true], permutation_map = #map6} : memref<?x?xf32>, vector<8xf32>
-// CHECK: vector.load %{{.*}}[%[[C0]], %[[C0]]] : memref<?x?xf32>, vector<1xf32>
+// CHECK: vector.load %{{.*}}[%[[C0]], %[[C0]]] {inbounds, nneg} : memref<?x?xf32>, vector<1xf32>
// CHECK: vector.broadcast %{{.*}} : vector<1xf32> to vector<8xf32>
return %0, %1, %2, %3, %4, %5, %6 : vector<7x14x8x16xf32>, vector<7x14x8x16xf32>,
>From 2a9be3ef63337c39c9b19fab25408b221ae17712 Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Tue, 16 Jun 2026 17:33:58 +0200
Subject: [PATCH 3/5] [mlir][VectorToLLVM] add opt-in `enableGEPInboundsNuw`
flag
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
mlir/include/mlir/Conversion/Passes.td | 6 ++
.../VectorToLLVM/ConvertVectorToLLVM.h | 2 +-
.../mlir/Dialect/Vector/IR/VectorOps.td | 65 ++-------------
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 37 +++++----
.../VectorToLLVM/ConvertVectorToLLVMPass.cpp | 2 +-
.../Vector/Transforms/LowerVectorGather.cpp | 6 +-
.../Vector/Transforms/LowerVectorTransfer.cpp | 17 +---
.../Conversion/GPUCommon/transfer_write.mlir | 2 +-
.../vector-to-llvm-interface.mlir | 80 ++-----------------
.../VectorToLLVM/vector-xfer-to-llvm.mlir | 4 +-
.../SuperVectorize/vectorize_2d_inbounds.mlir | 2 +-
.../vector-transfer-to-vector-load-store.mlir | 30 +++----
12 files changed, 68 insertions(+), 185 deletions(-)
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index c30dd3b07d028..c1c628cb1c1c9 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1614,6 +1614,12 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
"vector access are naturally aligned. If operations have an "
"alignment attribute set, the alignment attribute takes priority "
"over this option ">,
+ Option<"enableGEPInboundsNuw", "enable-gep-inbounds-nuw",
+ "bool", /*default=*/"false",
+ "Emit inbounds|nuw flags on GEPs generated by vector.load/store "
+ "lowering. Assumes 0 <= idx < dim_size and non-negative strides for "
+ "all dimensions. Setting this when those conditions do not hold is "
+ "undefined behavior under the LLVM lowering.">,
Option<"armNeon", "enable-arm-neon",
"bool", /*default=*/"false",
"Enables the use of ArmNeon dialect while lowering the vector "
diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
index cfb6cc313bc63..021a644ddd8a6 100644
--- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
+++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
@@ -17,7 +17,7 @@ class LLVMTypeConverter;
void populateVectorToLLVMConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns,
bool reassociateFPReductions = false, bool force32BitVectorIndices = false,
- bool useVectorAlignment = false);
+ bool useVectorAlignment = false, bool enableGEPInboundsNuw = false);
namespace vector {
void registerConvertVectorToLLVMInterface(DialectRegistry ®istry);
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index a98a14ae54d29..5acf2b4ab7649 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1730,58 +1730,32 @@ def Vector_LoadOp : Vector_Op<"load", [
load operation. It must be a positive power of 2. The operation must access
memory at an address aligned to this boundary. Violating this requirement
triggers immediate undefined behavior.
-
- Out-of-bounds behavior is implementation-defined at the `vector` level: both
- out-of-bounds start indices and the out-of-bounds tail of an in-bounds start
- (e.g. `%memref[%c0] : memref<7xf32>, vector<8xf32>`) have target-specific
- behavior (e.g. SPIR-V robust buffer access or AMDGPU buffer fat pointers may
- define them to read zeros). The optional `inbounds` and `nneg` unit
- attributes let a producer opt in to stronger assumptions used by lowerings:
- * `inbounds` asserts the access is in-bounds, i.e. `0 <= idx < dim_size`
- for every dimension. When set, the LLVM lowering may emit
- `llvm.getelementptr inbounds`.
- * `nneg` asserts that every index is non-negative. When set (and the
- memref has non-negative strides), the LLVM lowering may additionally
- emit the `nuw` flag.
-
- Both default to unset, in which case no flag is emitted and the
- target-defined out-of-bounds behavior above is preserved. Setting either
- attribute when it does not hold is undefined behavior under the LLVM
- lowering.
}];
let arguments = (ins Arg<AnyMemRef, "the reference to load from",
[MemRead]>:$base,
Variadic<Index>:$indices,
DefaultValuedOptionalAttr<BoolAttr, "false">:$nontemporal,
- OptionalAttr<IntValidAlignment<I64Attr>>: $alignment,
- UnitAttr:$inbounds,
- UnitAttr:$nneg);
+ OptionalAttr<IntValidAlignment<I64Attr>>: $alignment);
let builders = [
OpBuilder<(ins "VectorType":$resultType,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
- CArg<"bool", "false">:$inbounds,
- CArg<"bool", "false">:$nneg), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
return build($_builder, $_state, resultType, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr,
- inbounds, nneg);
+ nullptr);
}]>,
OpBuilder<(ins "TypeRange":$resultTypes,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
- CArg<"bool", "false">:$inbounds,
- CArg<"bool", "false">:$nneg), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
return build($_builder, $_state, resultTypes, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr,
- inbounds, nneg);
+ nullptr);
}]>
];
@@ -1873,24 +1847,6 @@ def Vector_StoreOp : Vector_Op<"store", [
store operation. It must be a positive power of 2. The operation must access
memory at an address aligned to this boundary. Violating this requirement
triggers immediate undefined behavior.
-
- Out-of-bounds behavior is implementation-defined at the `vector` level: both
- out-of-bounds start indices and the out-of-bounds tail of an in-bounds start
- (e.g. `%memref[%c0] : memref<7xf32>, vector<8xf32>`) have target-specific
- behavior (e.g. SPIR-V robust buffer access or AMDGPU buffer fat pointers may
- define them to drop the write). The optional `inbounds` and `nneg` unit
- attributes let a producer opt in to stronger assumptions used by lowerings:
- * `inbounds` asserts the access is in-bounds, i.e. `0 <= idx < dim_size`
- for every dimension. When set, the LLVM lowering may emit
- `llvm.getelementptr inbounds`.
- * `nneg` asserts that every index is non-negative. When set (and the
- memref has non-negative strides), the LLVM lowering may additionally
- emit the `nuw` flag.
-
- Both default to unset, in which case no flag is emitted and the
- target-defined out-of-bounds behavior above is preserved. Setting either
- attribute when it does not hold is undefined behavior under the LLVM
- lowering.
}];
let arguments = (ins
@@ -1899,22 +1855,17 @@ def Vector_StoreOp : Vector_Op<"store", [
[MemWrite]>:$base,
Variadic<Index>:$indices,
DefaultValuedOptionalAttr<BoolAttr, "false">:$nontemporal,
- OptionalAttr<IntValidAlignment<I64Attr>>: $alignment,
- UnitAttr:$inbounds,
- UnitAttr:$nneg);
+ OptionalAttr<IntValidAlignment<I64Attr>>: $alignment);
let builders = [
OpBuilder<(ins "Value":$valueToStore,
"Value":$base,
"ValueRange":$indices,
CArg<"bool", "false">:$nontemporal,
- CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment,
- CArg<"bool", "false">:$inbounds,
- CArg<"bool", "false">:$nneg), [{
+ CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
return build($_builder, $_state, valueToStore, base, indices, nontemporal,
alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
- nullptr,
- inbounds, nneg);
+ nullptr);
}]>
];
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 33d2d0bc8f055..9f444c5b0061c 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -239,10 +239,11 @@ template <class LoadOrStoreOp>
class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
public:
explicit VectorLoadStoreConversion(const LLVMTypeConverter &typeConv,
- bool useVectorAlign)
+ bool useVectorAlign,
+ bool enableGEPInboundsNuw)
: ConvertOpToLLVMPattern<LoadOrStoreOp>(typeConv),
- useVectorAlignment(useVectorAlign) {}
- using ConvertOpToLLVMPattern<LoadOrStoreOp>::ConvertOpToLLVMPattern;
+ useVectorAlignment(useVectorAlign),
+ enableGEPInboundsNuw(enableGEPInboundsNuw) {}
LogicalResult
matchAndRewrite(LoadOrStoreOp loadOrStoreOp,
@@ -266,21 +267,21 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
"could not resolve alignment");
// Resolve address.
- // `vector.load`/`vector.store` may carry `inbounds`/`nneg` assertions about
- // their indices (see the op docs). Translate them into GEP no-wrap flags so
- // LLVM can apply no-wrap optimizations on the generated index arithmetic
- // and GEP. When the attributes are absent (and for the masked variants,
- // which target near-boundary access) no flag is emitted, preserving any
- // target-defined out-of-bounds behavior.
+ // When --enable-gep-inbounds-nuw is set, emit inbounds|nuw on the GEP so
+ // LLVM can apply no-wrap optimizations on the index arithmetic. This
+ // assumes 0 <= idx < dim_size and non-negative strides; the caller is
+ // responsible for ensuring those conditions hold. Masked variants are
+ // designed for near-boundary access and never receive these flags.
LLVM::GEPNoWrapFlags noWrapFlags = LLVM::GEPNoWrapFlags::none;
if constexpr (std::is_same_v<LoadOrStoreOp, vector::LoadOp> ||
std::is_same_v<LoadOrStoreOp, vector::StoreOp>) {
- if (loadOrStoreOp.getInbounds())
+ if (enableGEPInboundsNuw) {
noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::inbounds;
- // `nuw` additionally requires the whole offset computation to be
- // non-negative: non-negative indices (nneg) *and* non-negative strides.
- if (loadOrStoreOp.getNneg() && hasNonNegativeStrides(memRefTy))
- noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::nuw;
+ // `nuw` additionally requires non-negative strides; skip it when the
+ // memref has dynamic or negative strides to avoid emitting poison.
+ if (hasNonNegativeStrides(memRefTy))
+ noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::nuw;
+ }
}
auto vtype = cast<VectorType>(
this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
@@ -298,6 +299,7 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
// of the memref. This flag is intended for use with hardware
// backends that require alignment of vector operations.
const bool useVectorAlignment;
+ const bool enableGEPInboundsNuw;
};
/// Conversion pattern for a vector.gather.
@@ -2245,7 +2247,7 @@ void mlir::vector::populateVectorTransposeToFlatTranspose(
void mlir::populateVectorToLLVMConversionPatterns(
const LLVMTypeConverter &converter, RewritePatternSet &patterns,
bool reassociateFPReductions, bool force32BitVectorIndices,
- bool useVectorAlignment) {
+ bool useVectorAlignment, bool enableGEPInboundsNuw) {
// This function populates only ConversionPatterns, not RewritePatterns.
MLIRContext *ctx = converter.getDialect()->getContext();
patterns.add<VectorReductionOpConversion>(converter, reassociateFPReductions);
@@ -2253,8 +2255,9 @@ void mlir::populateVectorToLLVMConversionPatterns(
patterns.add<VectorLoadStoreConversion<vector::LoadOp>,
VectorLoadStoreConversion<vector::MaskedLoadOp>,
VectorLoadStoreConversion<vector::StoreOp>,
- VectorLoadStoreConversion<vector::MaskedStoreOp>,
- VectorGatherOpConversion, VectorScatterOpConversion>(
+ VectorLoadStoreConversion<vector::MaskedStoreOp>>(
+ converter, useVectorAlignment, enableGEPInboundsNuw);
+ patterns.add<VectorGatherOpConversion, VectorScatterOpConversion>(
converter, useVectorAlignment);
patterns.add<VectorBitCastOpConversion, VectorShuffleOpConversion,
VectorExtractOpConversion, VectorFMAOp1DConversion,
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 4cc5704353382..2b358d312dcfe 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -114,7 +114,7 @@ void ConvertVectorToLLVMPass::runOnOperation() {
populateVectorTransferLoweringPatterns(patterns);
populateVectorToLLVMConversionPatterns(
converter, patterns, reassociateFPReductions, force32BitVectorIndices,
- useVectorAlignment);
+ useVectorAlignment, enableGEPInboundsNuw);
// Architecture specific augmentations.
LLVMConversionTarget target(getContext());
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index 111074dff9191..5a8e473d39360 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -293,9 +293,9 @@ struct Gather1DToConditionalLoads : OpRewritePattern<vector::GatherOp> {
if (isa<MemRefType>(base.getType())) {
// `vector.load` does not support scalar result; emit a vector load
// and extract the single result instead.
- Value load = vector::LoadOp::create(
- b, loc, elemVecTy, base, loadOffsets, nontemporalAttr,
- alignmentAttr, /*inbounds=*/nullptr, /*nneg=*/nullptr);
+ Value load =
+ vector::LoadOp::create(b, loc, elemVecTy, base, loadOffsets,
+ nontemporalAttr, alignmentAttr);
int64_t zeroIdx[1] = {0};
extracted = vector::ExtractOp::create(b, loc, load, zeroIdx);
} else {
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
index 77f54f639efad..2cf8f0beaa4de 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp
@@ -474,15 +474,9 @@ struct TransferReadToVectorLoadLowering
rewriter, read.getLoc(), unbroadcastedVectorType, read.getBase(),
read.getIndices(), read.getMask(), fill);
} else {
- // This pattern only matches when no dimension is out-of-bounds (see the
- // `hasOutOfBoundsDim` check above), so the access is in-bounds and the
- // indices are non-negative. Propagate that as `inbounds`/`nneg` so the
- // LLVM lowering can emit the corresponding GEP no-wrap flags.
res = vector::LoadOp::create(rewriter, read.getLoc(),
unbroadcastedVectorType, read.getBase(),
- read.getIndices(), /*nontemporal=*/false,
- /*alignment=*/llvm::MaybeAlign(),
- /*inbounds=*/true, /*nneg=*/true);
+ read.getIndices());
}
// Insert a broadcasting op if required.
@@ -576,15 +570,8 @@ struct TransferWriteToVectorStoreLowering
write.getIndices(), write.getMask(),
write.getVector());
} else {
- // This pattern only matches when no dimension is out-of-bounds (see the
- // `hasOutOfBoundsDim` check above), so the access is in-bounds and the
- // indices are non-negative. Propagate that as `inbounds`/`nneg` so the
- // LLVM lowering can emit the corresponding GEP no-wrap flags.
vector::StoreOp::create(rewriter, write.getLoc(), write.getVector(),
- write.getBase(), write.getIndices(),
- /*nontemporal=*/false,
- /*alignment=*/llvm::MaybeAlign(),
- /*inbounds=*/true, /*nneg=*/true);
+ write.getBase(), write.getIndices());
}
// There's no return value for StoreOps. Use Value() to signal success to
// matchAndRewrite.
diff --git a/mlir/test/Conversion/GPUCommon/transfer_write.mlir b/mlir/test/Conversion/GPUCommon/transfer_write.mlir
index 34dd28077eb47..4d2ae8c39240c 100644
--- a/mlir/test/Conversion/GPUCommon/transfer_write.mlir
+++ b/mlir/test/Conversion/GPUCommon/transfer_write.mlir
@@ -3,7 +3,7 @@
// CHECK-LABEL: @warp_extract
// CHECK-SAME: %[[VEC:[a-zA-Z0-9_]+]]: vector<1xf32>
// CHECK:%[[BASE:[0-9]+]] = llvm.extractvalue
-// CHECK:%[[PTR:[0-9]+]] = llvm.getelementptr inbounds|nuw %[[BASE]]
+// CHECK:%[[PTR:[0-9]+]] = llvm.getelementptr %[[BASE]]
// CHECK:llvm.store %[[VEC]], %[[PTR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
func.func @warp_extract(%arg0: index, %arg1: memref<1024x1024xf32>, %arg2: vector<1xf32>) {
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 63f9e7e3a4016..b6fe9a4643130 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1806,84 +1806,20 @@ func.func @store_with_alignment(%memref : memref<200x100xf32>, %i : index, %j :
// -----
//===----------------------------------------------------------------------===//
-// vector.load / vector.store inbounds / nneg flags
+// vector.load / vector.store with --enable-gep-inbounds-nuw pass option
//===----------------------------------------------------------------------===//
-// `inbounds` alone lowers to `getelementptr inbounds` (which implies nusw, hence
-// `nsw` on the index arithmetic). No `nuw` without `nneg`.
-func.func @load_inbounds(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] {inbounds} : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load_inbounds
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-// `inbounds` + `nneg` on a non-negative-strided memref lowers to
-// `getelementptr inbounds|nuw` and `overflow<nsw, nuw>`.
-func.func @load_inbounds_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] {inbounds, nneg} : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load_inbounds_nneg
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-// `nneg` alone (no `inbounds`) lowers to `getelementptr nuw` / `overflow<nuw>`.
-// This is the out-of-bounds-but-non-negative case (e.g. AMDGPU buffer fat
-// pointers), where we must not claim `inbounds`.
-func.func @load_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] {nneg} : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
+// RUN: mlir-opt %s -convert-vector-to-llvm='enable-gep-inbounds-nuw=1' -split-input-file | FileCheck %s --check-prefix=CHECK-INBOUNDS
-// CHECK-LABEL: func @load_nneg
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-// `nneg` must not produce `nuw` when a stride is negative: `mul nuw idx, stride`
-// would wrap. The negative major stride here disables `nuw` (and `inbounds` is
-// not requested), so a plain GEP is emitted.
-func.func @load_nneg_negative_stride(%memref : memref<200x100xf32, strided<[-100, 1], offset: ?>>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] {nneg} : memref<200x100xf32, strided<[-100, 1], offset: ?>>, vector<8xf32>
+// CHECK-INBOUNDS-LABEL: func @load_enable_gep_inbounds_nuw
+// CHECK-INBOUNDS: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
+// CHECK-INBOUNDS: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK-INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+func.func @load_enable_gep_inbounds_nuw(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
return %0 : vector<8xf32>
}
-// CHECK-LABEL: func @load_nneg_negative_stride
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-// Store side: `inbounds` + `nneg` lowers to `getelementptr inbounds|nuw`.
-func.func @store_inbounds_nneg(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<4xf32>
- vector.store %val, %memref[%i, %j] {inbounds, nneg} : memref<200x100xf32>, vector<4xf32>
- return
-}
-
-// CHECK-LABEL: func @store_inbounds_nneg
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
-
// -----
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
index 1d998e09212b4..18deadd0d7a79 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
@@ -298,7 +298,7 @@ func.func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
@@ -314,7 +314,7 @@ func.func @transfer_read_1d_inbounds_scalable(%A : memref<?xf32>, %base: index)
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
index 10a96654c2b36..49ba1a0c31ee1 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir
@@ -8,7 +8,7 @@
// RUN: | FileCheck %s --check-prefix=MATMUL
// RUN: mlir-opt %s \
// RUN: --affine-super-vectorize="virtual-vector-size=4" \
-// RUN: --convert-vector-to-llvm \
+// RUN: --convert-vector-to-llvm='enable-gep-inbounds-nuw=1' \
// RUN: --finalize-memref-to-llvm \
// RUN: --convert-func-to-llvm \
// RUN: | FileCheck %s --check-prefix=LLVM
diff --git a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
index 10b01ead1e717..8206c1a3ec865 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
@@ -6,10 +6,10 @@
func.func @vector_transfer_ops_0d_memref(%mem: memref<f32>, %vec: vector<f32>) {
%f0 = arith.constant 0.0 : f32
-// CHECK-NEXT: %[[S:.*]] = vector.load %[[MEM]][] {inbounds, nneg} : memref<f32>, vector<f32>
+// CHECK-NEXT: %[[S:.*]] = vector.load %[[MEM]][] : memref<f32>, vector<f32>
%0 = vector.transfer_read %mem[], %f0 : memref<f32>, vector<f32>
-// CHECK-NEXT: vector.store %[[S]], %[[MEM]][] {inbounds, nneg} : memref<f32>, vector<f32>
+// CHECK-NEXT: vector.store %[[S]], %[[MEM]][] : memref<f32>, vector<f32>
vector.transfer_write %0, %mem[] : vector<f32>, memref<f32>
// CHECK-NEXT: vector.store %[[VEC]], %[[MEM]][] : memref<f32>, vector<f32>
@@ -36,8 +36,8 @@ func.func @vector_transfer_ops_0d_tensor(%src: tensor<f32>) -> vector<1xf32> {
// CHECK-LABEL: func @transfer_to_load(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -69,8 +69,8 @@ func.func @masked_transfer_to_load(%mem : memref<8x8xf32>, %idx : index, %mask :
// CHECK-LABEL: func @transfer_2D(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<2x4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<2x4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<2x4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<2x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<2x4xf32>
// CHECK-NEXT: }
@@ -85,8 +85,8 @@ func.func @transfer_2D(%mem : memref<8x8xf32>, %idx : index) -> vector<2x4xf32>
// CHECK-LABEL: func @transfer_vector_element(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xvector<2x4xf32>>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<2x4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
-// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
+// CHECK-NEXT: vector.store %[[RES:.*]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xvector<2x4xf32>>, vector<2x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<2x4xf32>
// CHECK-NEXT: }
@@ -154,8 +154,8 @@ func.func @transfer_not_inbounds(%mem : memref<8x8xf32>, %idx : index) -> vector
// CHECK-LABEL: func @transfer_nondefault_layout(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32, #{{.*}}>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
-// CHECK-NEXT: vector.store %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32, #{{.*}}>, vector<4xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32, #{{.*}}>, vector<4xf32>
+// CHECK-NEXT: vector.store %[[RES]], %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32, #{{.*}}>, vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -188,7 +188,7 @@ func.func @transfer_perm_map(%mem : memref<8x8xf32>, %idx : index) -> vector<4xf
// CHECK-LABEL: func @transfer_broadcasting(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<1xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<1xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<1xf32> to vector<4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4xf32>
// CHECK-NEXT: }
@@ -205,7 +205,7 @@ func.func @transfer_broadcasting(%mem : memref<8x8xf32>, %idx : index) -> vector
// CHECK-LABEL: func @transfer_scalar(
// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<1xf32> {
-// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<?x?xf32>, vector<1xf32>
+// CHECK-NEXT: %[[RES:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<?x?xf32>, vector<1xf32>
// CHECK-NEXT: return %[[RES]] : vector<1xf32>
// CHECK-NEXT: }
func.func @transfer_scalar(%mem : memref<?x?xf32>, %idx : index) -> vector<1xf32> {
@@ -218,7 +218,7 @@ func.func @transfer_scalar(%mem : memref<?x?xf32>, %idx : index) -> vector<1xf32
// CHECK-LABEL: func @transfer_broadcasting_2D(
// CHECK-SAME: %[[MEM:.*]]: memref<8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<4x4xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<8x8xf32>, vector<1x1xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]]] : memref<8x8xf32>, vector<1x1xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<1x1xf32> to vector<4x4xf32>
// CHECK-NEXT: return %[[RES]] : vector<4x4xf32>
// CHECK-NEXT: }
@@ -236,7 +236,7 @@ func.func @transfer_broadcasting_2D(%mem : memref<8x8xf32>, %idx : index) -> vec
// CHECK-LABEL: func @transfer_broadcasting_complex(
// CHECK-SAME: %[[MEM:.*]]: memref<10x20x30x8x8xf32>,
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<3x2x4x5xf32> {
-// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]] {inbounds, nneg} : memref<10x20x30x8x8xf32>, vector<3x1x1x5xf32>
+// CHECK-NEXT: %[[LOAD:.*]] = vector.load %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]] : memref<10x20x30x8x8xf32>, vector<3x1x1x5xf32>
// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[LOAD]] : vector<3x1x1x5xf32> to vector<3x2x4x5xf32>
// CHECK-NEXT: return %[[RES]] : vector<3x2x4x5xf32>
// CHECK-NEXT: }
@@ -318,7 +318,7 @@ func.func @transfer_read_permutations(%mem_0 : memref<?x?xf32>, %mem_1 : memref<
// CHECK: vector.transpose %{{.*}}, [2, 1, 3, 0] : vector<16x14x7x8xf32> to vector<7x14x8x16xf32>
%6 = vector.transfer_read %mem_0[%c0, %c0], %cst {in_bounds = [true], permutation_map = #map6} : memref<?x?xf32>, vector<8xf32>
-// CHECK: vector.load %{{.*}}[%[[C0]], %[[C0]]] {inbounds, nneg} : memref<?x?xf32>, vector<1xf32>
+// CHECK: vector.load %{{.*}}[%[[C0]], %[[C0]]] : memref<?x?xf32>, vector<1xf32>
// CHECK: vector.broadcast %{{.*}} : vector<1xf32> to vector<8xf32>
return %0, %1, %2, %3, %4, %5, %6 : vector<7x14x8x16xf32>, vector<7x14x8x16xf32>,
>From 82748faa22a07091754da452035b62a511362c4c Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Wed, 17 Jun 2026 16:26:56 +0200
Subject: [PATCH 4/5] Address comments, hopefully :'D
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
.../VectorToLLVM/ConvertVectorToLLVM.cpp | 6 +
.../vector-load-store-to-llvm.mlir | 238 +++++++++++++++++
.../vector-to-llvm-interface.mlir | 245 ------------------
3 files changed, 244 insertions(+), 245 deletions(-)
create mode 100644 mlir/test/Conversion/VectorToLLVM/vector-load-store-to-llvm.mlir
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 9f444c5b0061c..8e9d37648841a 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -275,6 +275,12 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
LLVM::GEPNoWrapFlags noWrapFlags = LLVM::GEPNoWrapFlags::none;
if constexpr (std::is_same_v<LoadOrStoreOp, vector::LoadOp> ||
std::is_same_v<LoadOrStoreOp, vector::StoreOp>) {
+ // The verifier (verifyLoadStoreMemRefLayout) guarantees that the
+ // trailing (most minor) stride of the memref is 1. Assert to make
+ // the invariant explicit in the lowering code.
+ auto [strides, offset] = memRefTy.getStridesAndOffset();
+ assert((strides.empty() || strides.back() == 1) &&
+ "vector.load/store requires unit trailing memref stride");
if (enableGEPInboundsNuw) {
noWrapFlags = noWrapFlags | LLVM::GEPNoWrapFlags::inbounds;
// `nuw` additionally requires non-negative strides; skip it when the
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-load-store-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-load-store-to-llvm.mlir
new file mode 100644
index 0000000000000..cab462efee2b5
--- /dev/null
+++ b/mlir/test/Conversion/VectorToLLVM/vector-load-store-to-llvm.mlir
@@ -0,0 +1,238 @@
+// RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck %s --check-prefixes=ALL,DEFAULT
+// RUN: mlir-opt %s -convert-vector-to-llvm='enable-gep-inbounds-nuw=1' -split-input-file | FileCheck %s --check-prefixes=ALL,INBOUNDS
+
+//===----------------------------------------------------------------------===//
+// vector.load
+//===----------------------------------------------------------------------===//
+
+func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// ALL-LABEL: func @load
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<[8]xf32>
+ return %0 : vector<[8]xf32>
+}
+
+// ALL-LABEL: func @load_scalable
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
+
+// -----
+
+func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// ALL-LABEL: func @load_nontemporal
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
+ %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[8]xf32>
+ return %0 : vector<[8]xf32>
+}
+
+// ALL-LABEL: func @load_nontemporal_scalable
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
+
+// -----
+
+func.func @load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<8xindex>
+ return %0 : vector<8xindex>
+}
+// ALL-LABEL: func @load_index
+// ALL: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xi64>
+// ALL: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<8xi64> to vector<8xindex>
+// ALL: return %[[T1]] : vector<8xindex>
+
+// -----
+
+func.func @load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<[8]xindex>
+ return %0 : vector<[8]xindex>
+}
+// ALL-LABEL: func @load_index_scalable
+// ALL: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<[8]xi64>
+// ALL: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<[8]xi64> to vector<[8]xindex>
+// ALL: return %[[T1]] : vector<[8]xindex>
+
+// -----
+
+func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
+ %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<f32>
+ return %0 : vector<f32>
+}
+
+// ALL-LABEL: func @load_0d
+// ALL: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// ALL: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// ALL: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// ALL: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]]
+// DEFAULT: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
+// ALL: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
+// ALL: return %[[RES]] : vector<f32>
+
+// -----
+
+func.func @load_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+ %0 = vector.load %memref[%i, %j] { alignment = 8 } : memref<200x100xf32>, vector<8xf32>
+ return %0 : vector<8xf32>
+}
+
+// ALL-LABEL: func @load_with_alignment
+// ALL: llvm.load {{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.store
+//===----------------------------------------------------------------------===//
+
+func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<4xf32>
+ vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<4xf32>
+ return
+}
+
+// ALL-LABEL: func @store
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<[4]xf32>
+ vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<[4]xf32>
+ return
+}
+
+// ALL-LABEL: func @store_scalable
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<[4]xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<4xf32>
+ vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<4xf32>
+ return
+}
+
+// ALL-LABEL: func @store_nontemporal
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<4xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<[4]xf32>
+ vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[4]xf32>
+ return
+}
+
+// ALL-LABEL: func @store_nontemporal_scalable
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}}
+// DEFAULT: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<[4]xf32>, !llvm.ptr
+
+// -----
+
+func.func @store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
+ %val = arith.constant dense<11> : vector<4xindex>
+ vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<4xindex>
+ return
+}
+// ALL-LABEL: func @store_index
+// ALL: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<4xi64>, !llvm.ptr
+
+// -----
+
+func.func @store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
+ %val = arith.constant dense<11> : vector<[4]xindex>
+ vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<[4]xindex>
+ return
+}
+// ALL-LABEL: func @store_index_scalable
+// ALL: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[4]xi64>, !llvm.ptr
+
+// -----
+
+func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
+ %val = arith.constant dense<11.0> : vector<f32>
+ vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<f32>
+ return
+}
+
+// ALL-LABEL: func @store_0d
+// ALL: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// ALL: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
+// ALL: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// ALL: %[[CST:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
+// ALL: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
+// ALL: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// ALL: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
+// ALL: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]]
+// ALL: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]]
+// DEFAULT: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// INBOUNDS: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// ALL: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
+// ALL: return
+
+// -----
+
+func.func @store_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index, %val : vector<4xf32>) {
+ vector.store %val, %memref[%i, %j] {alignment = 8} : memref<200x100xf32>, vector<4xf32>
+ return
+}
+
+// ALL-LABEL: func @store_with_alignment
+// ALL: llvm.store %{{.*}} {alignment = 8 : i64} : vector<4xf32>, !llvm.ptr
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index b6fe9a4643130..e82f75d93066a 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1577,251 +1577,6 @@ func.func @transpose_0d(%arg0: vector<f32>) -> vector<f32> {
// -----
-//===----------------------------------------------------------------------===//
-// vector.load
-//===----------------------------------------------------------------------===//
-
-func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<[8]xf32>
- return %0 : vector<[8]xf32>
-}
-
-// CHECK-LABEL: func @load_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
-
-// -----
-
-func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load_nontemporal
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<[8]xf32> {
- %0 = vector.load %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[8]xf32>
- return %0 : vector<[8]xf32>
-}
-
-// CHECK-LABEL: func @load_nontemporal_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
-
-// -----
-
-func.func @load_index(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<8xindex> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<8xindex>
- return %0 : vector<8xindex>
-}
-// CHECK-LABEL: func @load_index
-// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xi64>
-// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<8xi64> to vector<8xindex>
-// CHECK: return %[[T1]] : vector<8xindex>
-
-// -----
-
-func.func @load_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) -> vector<[8]xindex> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xindex>, vector<[8]xindex>
- return %0 : vector<[8]xindex>
-}
-// CHECK-LABEL: func @load_index_scalable
-// CHECK: %[[T0:.*]] = llvm.load %{{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<[8]xi64>
-// CHECK: %[[T1:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<[8]xi64> to vector<[8]xindex>
-// CHECK: return %[[T1]] : vector<[8]xindex>
-
-// -----
-
-func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<f32> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<f32>
- return %0 : vector<f32>
-}
-
-// CHECK-LABEL: func @load_0d
-// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
-// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
-// CHECK: return %[[RES]] : vector<f32>
-
-// -----
-
-func.func @load_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] { alignment = 8 } : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// CHECK-LABEL: func @load_with_alignment
-// CHECK: llvm.load {{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xf32>
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.store
-//===----------------------------------------------------------------------===//
-
-func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<4xf32>
- vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<4xf32>
- return
-}
-
-// CHECK-LABEL: func @store
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<[4]xf32>
- vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<[4]xf32>
- return
-}
-
-// CHECK-LABEL: func @store_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<[4]xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<4xf32>
- vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<4xf32>
- return
-}
-
-// CHECK-LABEL: func @store_nontemporal
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<4xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<[4]xf32>
- vector.store %val, %memref[%i, %j] {nontemporal = true} : memref<200x100xf32>, vector<[4]xf32>
- return
-}
-
-// CHECK-LABEL: func @store_nontemporal_scalable
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<[4]xf32>, !llvm.ptr
-
-// -----
-
-func.func @store_index(%memref : memref<200x100xindex>, %i : index, %j : index) {
- %val = arith.constant dense<11> : vector<4xindex>
- vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<4xindex>
- return
-}
-// CHECK-LABEL: func @store_index
-// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<4xi64>, !llvm.ptr
-
-// -----
-
-func.func @store_index_scalable(%memref : memref<200x100xindex>, %i : index, %j : index) {
- %val = arith.constant dense<11> : vector<[4]xindex>
- vector.store %val, %memref[%i, %j] : memref<200x100xindex>, vector<[4]xindex>
- return
-}
-// CHECK-LABEL: func @store_index_scalable
-// CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[4]xi64>, !llvm.ptr
-
-// -----
-
-func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
- %val = arith.constant dense<11.0> : vector<f32>
- vector.store %val, %memref[%i, %j] : memref<200x100xf32>, vector<f32>
- return
-}
-
-// CHECK-LABEL: func @store_0d
-// CHECK: %[[J:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[I:.*]] = builtin.unrealized_conversion_cast %{{.*}} : index to i64
-// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[CST:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
-// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
-// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
-// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
-// CHECK: return
-
-// -----
-
-func.func @store_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index, %val : vector<4xf32>) {
- vector.store %val, %memref[%i, %j] {alignment = 8} : memref<200x100xf32>, vector<4xf32>
- return
-}
-
-// CHECK-LABEL: func @store_with_alignment
-// CHECK: llvm.store %{{.*}} {alignment = 8 : i64} : vector<4xf32>, !llvm.ptr
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// vector.load / vector.store with --enable-gep-inbounds-nuw pass option
-//===----------------------------------------------------------------------===//
-
-// RUN: mlir-opt %s -convert-vector-to-llvm='enable-gep-inbounds-nuw=1' -split-input-file | FileCheck %s --check-prefix=CHECK-INBOUNDS
-
-// CHECK-INBOUNDS-LABEL: func @load_enable_gep_inbounds_nuw
-// CHECK-INBOUNDS: %[[MUL:.*]] = llvm.mul %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
-// CHECK-INBOUNDS: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
-// CHECK-INBOUNDS: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-func.func @load_enable_gep_inbounds_nuw(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
- %0 = vector.load %memref[%i, %j] : memref<200x100xf32>, vector<8xf32>
- return %0 : vector<8xf32>
-}
-
-// -----
-
//===----------------------------------------------------------------------===//
// vector.maskedload
//===----------------------------------------------------------------------===//
>From c124a6153a0abcb56a6aa9cb77f5dafb1cac3aa9 Mon Sep 17 00:00:00 2001
From: Federico Bruzzone <federico.bruzzone.i at gmail.com>
Date: Thu, 18 Jun 2026 08:37:21 +0200
Subject: [PATCH 5/5] Affine-super-vectorizer docs
Signed-off-by: Federico Bruzzone <federico.bruzzone.i at gmail.com>
---
.../include/mlir/Dialect/Affine/Transforms/Passes.td | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/mlir/include/mlir/Dialect/Affine/Transforms/Passes.td b/mlir/include/mlir/Dialect/Affine/Transforms/Passes.td
index 1c54ca7deca91..d3b5e6306b367 100644
--- a/mlir/include/mlir/Dialect/Affine/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Affine/Transforms/Passes.td
@@ -345,6 +345,18 @@ def AffineScalarReplacement : Pass<"affine-scalrep", "func::FuncOp"> {
def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
let summary = "Vectorize to a target independent n-D vector abstraction";
+ let description = [{
+ Vectorizes affine loops into `vector.transfer_read` / `vector.transfer_write`
+ operations. When an access is provably within bounds (all loop bounds are
+ statically known and the access pattern fits the memref), the generated
+ transfer ops carry `in_bounds = [true, ...]`, asserting that no
+ out-of-bounds masking is needed.
+
+ When this pass is combined with `ConvertVectorToLLVM` and its
+ `enable-gep-inbounds-nuw` option is set, the lowering of those
+ `vector.load` / `vector.store` ops (produced by lowering the
+ in-bounds transfers) emits `llvm.getelementptr inbounds|nuw`.
+ }];
let dependentDialects = ["vector::VectorDialect"];
let options = [
ListOption<"vectorSizes", "virtual-vector-size", "int64_t",
More information about the Mlir-commits
mailing list