[Mlir-commits] [mlir] [mlir][VectorToLLVM] emit `inbounds|nuw` GEP flags when lowering `vector.load/store` (PR #202118)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Sun Jun 7 04:04:29 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir
Author: Federico Bruzzone (FedericoBruzzone)
<details>
<summary>Changes</summary>
This patch follows up on #<!-- -->201180, which fixed `affine-super-vectorize` to emit `in_bounds = [true]` on `vector.transfer_read/write` when accesses are statically provable to be within bounds. Even with that fix in place, the generated LLVM IR was still suboptimal: the subsequent VectorToLLVM lowering emitted `llvm.getelementptr` without `inbounds` or `nuw` flags, leaving LLVM's optimizer unable to exploit the no-wrap guarantee.
**Root Cause**
VectorLoadStoreConversion calls getStridedElementPtr with the default `GEPNoWrapFlags::none`. The equivalent MemRefToLLVM lowering for `memref.load/memref.store` already passes `GEPNoWrapFlags::inbounds | GEPNoWrapFlags::nuw`, as mandated by the spec (0 <= idx < dim_size). `vector.load/vector.store` carry the same guarantee, so the same flags are correct.
The missing flags have two concrete downstream effects:
- Index arithmetic (`llvm.mul/llvm.add`): with no `overflow<nsw,nuw>` annotation SCEV cannot prove the induction-variable computation is monotone, thus loop auto-vectorizer bails out.
- GEP: with no `inbounds|nuw`the BasicAliasAnalysis falls back to conservative aliasing, thus further vectorization and alias-based optimizations are blocked.
Masked variants (`vector.maskedload/vector.maskedstore`) conservatively retain `GEPNoWrapFlags::none`.
---
Patch is 33.00 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/202118.diff
12 Files Affected:
- (modified) mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp (+11-2)
- (modified) mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp (+30-2)
- (modified) mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir (+2-2)
- (modified) mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir (+30-30)
- (modified) mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir (+2-2)
- (modified) mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir (+1-1)
- (modified) mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir (+4-4)
- (modified) mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir (+2-2)
- (added) mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d_inbounds.mlir (+56)
- (modified) mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir (+6-6)
- (added) mlir/test/Dialect/Affine/SuperVectorize/vectorize_inbounds_llvmopt.mlir (+49)
- (modified) mlir/test/Dialect/Affine/SuperVectorize/vectorize_reduction.mlir (+3-3)
``````````diff
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 43e0824fef6cd..0ba48b1f3862e 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -256,10 +256,19 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
"could not resolve alignment");
// Resolve address.
+ // Per vector.load/store spec, indices must be in-bounds (0 <= idx <
+ // dim_size). Emit inbounds|nuw so LLVM can apply no-wrap optimizations on
+ // the generated index arithmetic and GEP. Masked variants are designed for
+ // near-boundary access, so they conservatively omit these flags.
+ LLVM::GEPNoWrapFlags noWrapFlags = LLVM::GEPNoWrapFlags::none;
+ if constexpr (std::is_same_v<LoadOrStoreOp, vector::LoadOp> ||
+ std::is_same_v<LoadOrStoreOp, vector::StoreOp>)
+ noWrapFlags = LLVM::GEPNoWrapFlags::inbounds | LLVM::GEPNoWrapFlags::nuw;
auto vtype = cast<VectorType>(
this->typeConverter->convertType(loadOrStoreOp.getVectorType()));
- Value dataPtr = this->getStridedElementPtr(
- rewriter, loc, memRefTy, adaptor.getBase(), adaptor.getIndices());
+ Value dataPtr =
+ this->getStridedElementPtr(rewriter, loc, memRefTy, adaptor.getBase(),
+ adaptor.getIndices(), noWrapFlags);
replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, dataPtr, align,
rewriter);
return success();
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index c90d9bd8730e6..2027b389c02d3 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -1220,6 +1220,28 @@ static bool isIVMappedToMultipleIndices(
return false;
}
+/// Returns an in-bounds mask for a transfer op given its permutation map and
+/// the memref being accessed. Dimension i is in-bounds when the map result is
+/// an AffineDimExpr pointing to a static memref dimension that is divisible by
+/// the vector size, or an AffineConstantExpr.
+static SmallVector<bool> computeInBoundsMask(AffineMap permutationMap,
+ VectorType vectorType,
+ MemRefType memrefType) {
+ SmallVector<bool> inBounds(vectorType.getRank(), false);
+ for (unsigned i = 0; i < vectorType.getRank(); ++i) {
+ AffineExpr expr = permutationMap.getResult(i);
+ if (auto dimExpr = dyn_cast<AffineDimExpr>(expr)) {
+ unsigned memDim = dimExpr.getPosition();
+ if (!memrefType.isDynamicDim(memDim) &&
+ memrefType.getDimSize(memDim) % vectorType.getDimSize(i) == 0)
+ inBounds[i] = true;
+ } else if (isa<AffineConstantExpr>(expr)) {
+ inBounds[i] = true;
+ }
+ }
+ return inBounds;
+}
+
/// Vectorizes an affine load with the vectorization strategy in 'state' by
/// generating a 'vector.transfer_read' op with the proper permutation map
/// inferred from the indices of the load. The new 'vector.transfer_read' is
@@ -1265,9 +1287,12 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
LLVM_DEBUG(permutationMap.print(dbgs()));
+ SmallVector<bool> inBounds =
+ computeInBoundsMask(permutationMap, vectorType,
+ cast<MemRefType>(loadOp.getMemRef().getType()));
auto transfer = vector::TransferReadOp::create(
state.builder, loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
- /*padding=*/std::nullopt, permutationMap);
+ /*padding=*/std::nullopt, permutationMap, ArrayRef<bool>(inBounds));
// Register replacement for future uses in the scope.
state.registerOpVectorReplacement(loadOp, transfer);
@@ -1321,9 +1346,12 @@ static Operation *vectorizeAffineStore(AffineStoreOp storeOp,
return nullptr;
}
+ auto vType = cast<VectorType>(vectorValue.getType());
+ SmallVector<bool> inBounds = computeInBoundsMask(
+ permutationMap, vType, cast<MemRefType>(storeOp.getMemRef().getType()));
auto transfer = vector::TransferWriteOp::create(
state.builder, storeOp.getLoc(), vectorValue, storeOp.getMemRef(),
- indices, permutationMap);
+ indices, permutationMap, ArrayRef<bool>(inBounds));
LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ vectorized store: " << transfer);
// Register replacement for future uses in the scope.
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
index 80e6caa05db5e..58fb69e03c85b 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-scalable-memcpy.mlir
@@ -12,11 +12,11 @@ func.func @vector_scalable_memcopy(%src : memref<?xf32>, %dst : memref<?xf32>, %
scf.for %i0 = %c0 to %size step %step {
// CHECK: [[DATAIDX:%[0-9]+]] = builtin.unrealized_conversion_cast [[LOOPIDX]] : index to i64
// CHECK: [[SRCMEM:%[0-9]+]] = llvm.extractvalue [[SRCMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[SRCPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[SRCMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: [[LDVAL:%[0-9]+]] = llvm.load [[SRCPTR]]{{.*}}: !llvm.ptr -> vector<[4]xf32>
%0 = vector.load %src[%i0] : memref<?xf32>, vector<[4]xf32>
// CHECK: [[DSTMEM:%[0-9]+]] = llvm.extractvalue [[DSTMRS]][1] : !llvm.struct<(ptr
- // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK-NEXT: [[DSTPTR:%[0-9]+]] = llvm.getelementptr inbounds|nuw [[DSTMEM]]{{.}}[[DATAIDX]]{{.}} : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK-NEXT: llvm.store [[LDVAL]], [[DSTPTR]]{{.*}}: vector<[4]xf32>, !llvm.ptr
vector.store %0, %dst[%i0] : memref<?xf32>, vector<[4]xf32>
}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index d570d46e11b4a..0dda2333f962f 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1588,9 +1588,9 @@ func.func @load(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector
// CHECK-LABEL: func @load
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1602,9 +1602,9 @@ func.func @load_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @load_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1616,9 +1616,9 @@ func.func @load_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : inde
// CHECK-LABEL: func @load_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<8xf32>
// -----
@@ -1630,9 +1630,9 @@ func.func @load_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @load_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.load %[[GEP]] {alignment = 4 : i64, nontemporal} : !llvm.ptr -> vector<[8]xf32>
// -----
@@ -1670,9 +1670,9 @@ func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vec
// CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[LOAD:.*]] = llvm.load %[[ADDR]] {alignment = 4 : i64} : !llvm.ptr -> vector<1xf32>
// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[LOAD]] : vector<1xf32> to vector<f32>
// CHECK: return %[[RES]] : vector<f32>
@@ -1701,9 +1701,9 @@ func.func @store(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK-LABEL: func @store
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<4xf32>, !llvm.ptr
// -----
@@ -1716,9 +1716,9 @@ func.func @store_scalable(%memref : memref<200x100xf32>, %i : index, %j : index)
// CHECK-LABEL: func @store_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1731,9 +1731,9 @@ func.func @store_nontemporal(%memref : memref<200x100xf32>, %i : index, %j : ind
// CHECK-LABEL: func @store_nontemporal
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<4xf32>, !llvm.ptr
// -----
@@ -1746,9 +1746,9 @@ func.func @store_nontemporal_scalable(%memref : memref<200x100xf32>, %i : index,
// CHECK-LABEL: func @store_nontemporal_scalable
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} : i64
-// CHECK: %[[GEP:.*]] = llvm.getelementptr %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %{{.*}}, %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %{{.*}} overflow<nsw, nuw> : i64
+// CHECK: %[[GEP:.*]] = llvm.getelementptr inbounds|nuw %{{.*}}[%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %{{.*}}, %[[GEP]] {alignment = 4 : i64, nontemporal} : vector<[4]xf32>, !llvm.ptr
// -----
@@ -1787,9 +1787,9 @@ func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
// CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
// CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
-// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
-// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
-// CHECK: %[[ADDR:.*]] = llvm.getelementptr %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] overflow<nsw, nuw> : i64
+// CHECK: %[[ADDR:.*]] = llvm.getelementptr inbounds|nuw %[[REF]][%[[ADD]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: llvm.store %[[VAL]], %[[ADDR]] {alignment = 4 : i64} : vector<1xf32>, !llvm.ptr
// CHECK: return
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
index 18deadd0d7a79..1d998e09212b4 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
@@ -298,7 +298,7 @@ func.func @transfer_read_1d_inbounds(%A : memref<?xf32>, %base: index) -> vector
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<17xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
@@ -314,7 +314,7 @@ func.func @transfer_read_1d_inbounds_scalable(%A : memref<?xf32>, %base: index)
// CHECK-SAME: %[[BASE:[a-zA-Z0-9]*]]: index) -> vector<[17]xf32>
//
// 1. Bitcast to vector form.
-// CHECK: %[[gep:.*]] = llvm.getelementptr {{.*}} :
+// CHECK: %[[gep:.*]] = llvm.getelementptr inbounds|nuw {{.*}} :
// CHECK-SAME: (!llvm.ptr, i64) -> !llvm.ptr, f32
//
// 2. Rewrite as a load.
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
index fcf31daa987b4..71d8314bb60d8 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
@@ -73,7 +73,7 @@ func.func @transfer_rank_mismatch_no_crash(%arg0: memref<82x97xf32>) {
// VECNEST: vector.transfer_read
// VECNEST-NEXT: affine.for %{{.*}} = 0 to 30 {
// VECNEST: vector.transfer_read
-// VECNEST-NEXT: vector.transfer_write %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] {permutation_map = #{{.*}}}
+// VECNEST-NEXT: vector.transfer_write %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] {in_bounds = [true], permutation_map = #{{.*}}}
// VECNEST-NEXT: }
// VECNEST-NEXT: vector.transfer_write
// VECNEST: }
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
index f9593221e1843..e47c1d186ce40 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
@@ -22,7 +22,7 @@ func.func @vec1d_1(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
// CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
// CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
// CHECK-NEXT: %{{.*}} = ub.poison : f32
-// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
affine.for %i0 = 0 to %M { // vectorized due to scalar -> vector
%a0 = affine.load %A[%c0, %c0] : memref<?x?xf32>
}
@@ -171,7 +171,7 @@ func.func @vec_block_arg(%A : memref<32x512xi32>) {
// CHECK-NEXT: affine.for %[[IV1:[0-9a-zA-Z_]+]] = 0 to 32 {
// CHECK-NEXT: %[[BROADCAST:.*]] = vector.broadcast %[[IV1]] : index to vector<128xindex>
// CHECK-NEXT: %[[CAST:.*]] = arith.index_cast %[[BROADCAST]] : vector<128xindex> to vector<128xi32>
- // CHECK-NEXT: vector.transfer_write %[[CAST]], {{.*}}[%[[IV1]], %[[IV0]]] : vector<128xi32>, memref<32x512xi32>
+ // CHECK-NEXT: vector.transfer_write %[[CAST]], {{.*}}[%[[IV1]], %[[IV0]]] {in_bounds = [true]} : vector<128xi32>, memref<32x512xi32>
affine.for %i = 0 to 512 { // vectorized
affine.for %j = 0 to 32 {
%idx = arith.index_cast %j : index to i32
@@ -425,7 +425,7 @@ func.func @vec_rejected_8(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
// CHECK: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
// CHECK: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
// CHECK: %{{.*}} = ub.poison : f32
-// CHECK: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %{{.*}} in DFS post-order prevents vectorizing %{{.*}}
affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
%a18 = affine.load %A[%c0, %c0] : memref<?x?xf32>
@@ -459,7 +459,7 @@ func.func @vec_rejected_9(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
// CHECK: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
// CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
// CHECK-NEXT: %{{.*}} = ub.poison : f32
-// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %i18 in DFS post-order prevents vectorizing %{{.*}}
affine.for %i18 = 0 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/202118
More information about the Mlir-commits
mailing list