[Mlir-commits] [mlir] [MLIR][LLVM] Add invariant intrinsics (PR #75354)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Wed Dec 13 08:57:09 PST 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-mlir-llvm
Author: Tobias Gysi (gysit)
<details>
<summary>Changes</summary>
This commit implements the LLVM IR invariant intrinsics in LLVM dialect. These intrinsics can be used to mark a program regions in which the contents of a specific memory object will not change.
The LLVM dialect implementation also implements the PromotableOpInterface to ensure Mem2Reg & SROA are able to promote pointers that are marked using the invariant intrinsics.
---
Full diff: https://github.com/llvm/llvm-project/pull/75354.diff
7 Files Affected:
- (modified) mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td (+24-3)
- (modified) mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp (+22)
- (modified) mlir/test/Dialect/LLVMIR/mem2reg.mlir (+2)
- (modified) mlir/test/Dialect/LLVMIR/roundtrip.mlir (+10)
- (modified) mlir/test/Dialect/LLVMIR/sroa.mlir (+5-3)
- (modified) mlir/test/Target/LLVMIR/Import/intrinsic.ll (+12)
- (modified) mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir (+12)
``````````diff
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index f8862045f12b84..649bfe832f3c21 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -274,9 +274,9 @@ def LLVM_NoAliasScopeDeclOp
let assemblyFormat = "$scope attr-dict";
}
-//===----------------------------------------------------------------------===//
-// Lifetime Markers
-//===----------------------------------------------------------------------===//
+//
+// Memory marker intrinsics.
+//
/// Base operation for lifetime markers. The LLVM intrinsics require the size
/// operand to be an immediate. In MLIR it is encoded as an attribute.
@@ -291,6 +291,27 @@ class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [1],
def LLVM_LifetimeStartOp : LLVM_LifetimeBaseOp<"lifetime.start">;
def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
+def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
+ [DeclareOpInterfaceMethods<PromotableOpInterface>],
+ /*requiresFastmath=*/0, /*immArgPositions=*/[0],
+ /*immArgAttrNames=*/["size"]> {
+ let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
+ let results = (outs LLVM_AnyPointer:$res);
+ let assemblyFormat = "$size `,` $ptr attr-dict `:` "
+ "`(` qualified(type($ptr)) `)` `->` qualified(type($res))";
+}
+
+def LLVM_InvariantEndOp : LLVM_ZeroResultIntrOp<"invariant.end", [2],
+ [DeclareOpInterfaceMethods<PromotableOpInterface>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ /*immArgPositions=*/[1], /*immArgAttrNames=*/["size"]> {
+ let arguments = (ins LLVM_AnyPointer:$start,
+ I64Attr:$size,
+ LLVM_AnyPointer:$ptr);
+ let assemblyFormat = "$start `,` $size `,` $ptr attr-dict `:` "
+ "qualified(type($start)) `,` qualified(type($ptr))";
+}
+
// Intrinsics with multiple returns.
class LLVM_ArithWithOverflowOp<string mnem>
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 43349a26ad1127..96a0ef591c1cfe 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -248,6 +248,28 @@ DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
return DeletionKind::Delete;
}
+bool LLVM::InvariantStartOp::canUsesBeRemoved(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses,
+ SmallVectorImpl<OpOperand *> &newBlockingUses) {
+ return true;
+}
+
+DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+ return DeletionKind::Delete;
+}
+
+bool LLVM::InvariantEndOp::canUsesBeRemoved(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses,
+ SmallVectorImpl<OpOperand *> &newBlockingUses) {
+ return true;
+}
+
+DeletionKind LLVM::InvariantEndOp::removeBlockingUses(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+ return DeletionKind::Delete;
+}
+
bool LLVM::DbgDeclareOp::canUsesBeRemoved(
const SmallPtrSetImpl<OpOperand *> &blockingUses,
SmallVectorImpl<OpOperand *> &newBlockingUses) {
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
index 16eb28c629ee23..fe718e6f54c652 100644
--- a/mlir/test/Dialect/LLVMIR/mem2reg.mlir
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -520,6 +520,8 @@ llvm.func @discardable_use_tree() {
%4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
llvm.intr.lifetime.start 2, %3 : !llvm.ptr
llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+ %5 = llvm.intr.invariant.start 2, %3 : (!llvm.ptr) -> !llvm.ptr
+ llvm.intr.invariant.end %5, 2, %3 : !llvm.ptr, !llvm.ptr
llvm.return
}
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 594c3de91815ae..dac777cd6d5c32 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -577,6 +577,16 @@ llvm.func @lifetime(%p: !llvm.ptr) {
llvm.return
}
+// CHECK-LABEL: @invariant
+// CHECK-SAME: %[[P:.*]]: !llvm.ptr
+llvm.func @invariant(%p: !llvm.ptr) {
+ // CHECK: %[[START:.*]] = llvm.intr.invariant.start 1, %[[P]] : (!llvm.ptr) -> !llvm.ptr
+ %1 = llvm.intr.invariant.start 1, %p : (!llvm.ptr) -> !llvm.ptr
+ // CHECK: llvm.intr.invariant.end %[[START]], 1, %[[P]] : !llvm.ptr, !llvm.ptr
+ llvm.intr.invariant.end %1, 1, %p : !llvm.ptr, !llvm.ptr
+ llvm.return
+}
+
// CHECK-LABEL: @vararg_func
llvm.func @vararg_func(%arg0: i32, ...) {
// CHECK: %[[C:.*]] = llvm.mlir.constant(1 : i32)
diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir
index 09ca94e91d2a12..80f2f5e23bc02a 100644
--- a/mlir/test/Dialect/LLVMIR/sroa.mlir
+++ b/mlir/test/Dialect/LLVMIR/sroa.mlir
@@ -170,12 +170,14 @@ llvm.func @direct_promotable_use_is_fine_on_accessor() -> i32 {
// CHECK: %[[ALLOCA:.*]] = llvm.alloca %[[SIZE]] x i32
%1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, f64, i32)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
%2 = llvm.getelementptr %1[0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, f64, i32)>
+ // This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
+ %3 = llvm.intr.invariant.start 2, %2 : (!llvm.ptr) -> !llvm.ptr
// CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
- %3 = llvm.load %2 : !llvm.ptr -> i32
+ %4 = llvm.load %2 : !llvm.ptr -> i32
// This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
- llvm.intr.lifetime.start 2, %2 : !llvm.ptr
+ llvm.intr.invariant.end %3, 2, %2 : !llvm.ptr, !llvm.ptr
// CHECK: llvm.return %[[RES]] : i32
- llvm.return %3 : i32
+ llvm.return %4 : i32
}
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index c8dcde11d93e64..19a772df9d5bb9 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -754,6 +754,15 @@ define void @lifetime(ptr %0) {
ret void
}
+; CHECK-LABEL: llvm.func @invariant
+define void @invariant(ptr %0) {
+ ; CHECK: %[[START:.*]] = llvm.intr.invariant.start 16, %{{.*}} : (!llvm.ptr) -> !llvm.ptr
+ %2 = call ptr @llvm.invariant.start.p0(i64 16, ptr %0)
+ ; CHECK: llvm.intr.invariant.end %[[START]], 32, %{{.*}} : !llvm.ptr, !llvm.ptr
+ call void @llvm.invariant.end.p0(ptr %2, i64 32, ptr %0)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @vector_insert
define void @vector_insert(<vscale x 4 x float> %0, <4 x float> %1) {
; CHECK: llvm.intr.vector.insert %{{.*}}, %{{.*}}[4] : vector<4xf32> into !llvm.vec<? x 4 x f32>
@@ -1096,6 +1105,9 @@ declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32)
declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32)
declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
declare void @llvm.assume(i1)
declare float @llvm.ssa.copy.f32(float returned)
declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index e586c0cd2720e4..20298fa90540bc 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -941,6 +941,15 @@ llvm.func @lifetime(%p: !llvm.ptr) {
llvm.return
}
+// CHECK-LABEL: @invariant
+llvm.func @invariant(%p: !llvm.ptr) {
+ // CHECK: call ptr @llvm.invariant.start
+ %1 = llvm.intr.invariant.start 16, %p : (!llvm.ptr) -> !llvm.ptr
+ // CHECK: call void @llvm.invariant.end
+ llvm.intr.invariant.end %1, 16, %p : !llvm.ptr, !llvm.ptr
+ llvm.return
+}
+
// CHECK-LABEL: @ssa_copy
llvm.func @ssa_copy(%arg: f32) -> f32 {
// CHECK: call float @llvm.ssa.copy
@@ -1101,6 +1110,9 @@ llvm.func @ssa_copy(%arg: f32) -> f32 {
// CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
// CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
// CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
// CHECK-DAG: declare float @llvm.ssa.copy.f32(float returned)
// CHECK-DAG: declare ptr @llvm.stacksave.p0()
// CHECK-DAG: declare ptr addrspace(1) @llvm.stacksave.p1()
``````````
</details>
https://github.com/llvm/llvm-project/pull/75354
More information about the Mlir-commits
mailing list