[Mlir-commits] [mlir] 25d9424 - [mlir][llvm] Add invariant intrinsics (#75354)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Thu Dec 14 05:58:50 PST 2023
Author: Tobias Gysi
Date: 2023-12-14T14:58:45+01:00
New Revision: 25d942403c4c1cdefefd1d2ca6506ae75b921316
URL: https://github.com/llvm/llvm-project/commit/25d942403c4c1cdefefd1d2ca6506ae75b921316
DIFF: https://github.com/llvm/llvm-project/commit/25d942403c4c1cdefefd1d2ca6506ae75b921316.diff
LOG: [mlir][llvm] Add invariant intrinsics (#75354)
This commit implements the LLVM IR invariant intrinsics in LLVM dialect.
These intrinsics can be used to mark a program regions in which the
contents of a specific memory object will not change.
The LLVM dialect implementation also implements the
PromotableOpInterface to ensure Mem2Reg & SROA are able to promote
pointers that are marked using the invariant intrinsics.
Added:
Modified:
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
mlir/test/Dialect/LLVMIR/mem2reg.mlir
mlir/test/Dialect/LLVMIR/roundtrip.mlir
mlir/test/Dialect/LLVMIR/sroa.mlir
mlir/test/Target/LLVMIR/Import/intrinsic.ll
mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index f8862045f12b84..a798cad6037776 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -274,9 +274,9 @@ def LLVM_NoAliasScopeDeclOp
let assemblyFormat = "$scope attr-dict";
}
-//===----------------------------------------------------------------------===//
-// Lifetime Markers
-//===----------------------------------------------------------------------===//
+//
+// Memory marker intrinsics.
+//
/// Base operation for lifetime markers. The LLVM intrinsics require the size
/// operand to be an immediate. In MLIR it is encoded as an attribute.
@@ -291,6 +291,26 @@ class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [1],
def LLVM_LifetimeStartOp : LLVM_LifetimeBaseOp<"lifetime.start">;
def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
+def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
+ [DeclareOpInterfaceMethods<PromotableOpInterface>],
+ /*requiresFastmath=*/0, /*immArgPositions=*/[0],
+ /*immArgAttrNames=*/["size"]> {
+ let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
+ let results = (outs LLVM_DefaultPointer:$res);
+ let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
+}
+
+def LLVM_InvariantEndOp : LLVM_ZeroResultIntrOp<"invariant.end", [2],
+ [DeclareOpInterfaceMethods<PromotableOpInterface>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ /*immArgPositions=*/[1], /*immArgAttrNames=*/["size"]> {
+ let arguments = (ins LLVM_DefaultPointer:$start,
+ I64Attr:$size,
+ LLVM_AnyPointer:$ptr);
+ let assemblyFormat = "$start `,` $size `,` $ptr attr-dict `:` "
+ "qualified(type($ptr))";
+}
+
// Intrinsics with multiple returns.
class LLVM_ArithWithOverflowOp<string mnem>
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index 2ce8d8df05edc4..b6aa73dad22970 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -67,6 +67,9 @@ class LLVM_PointerInAddressSpace<int addressSpace> : Type<
# addressSpace # ")";
}
+// Type constraint accepting an LLVM pointer type in address space 0.
+def LLVM_DefaultPointer : LLVM_PointerInAddressSpace<0>;
+
// Type constraint accepting any LLVM structure type.
def LLVM_AnyStruct : Type<CPred<"::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">,
"LLVM structure type">;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 43349a26ad1127..96a0ef591c1cfe 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -248,6 +248,28 @@ DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
return DeletionKind::Delete;
}
+bool LLVM::InvariantStartOp::canUsesBeRemoved(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses,
+ SmallVectorImpl<OpOperand *> &newBlockingUses) {
+ return true;
+}
+
+DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+ return DeletionKind::Delete;
+}
+
+bool LLVM::InvariantEndOp::canUsesBeRemoved(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses,
+ SmallVectorImpl<OpOperand *> &newBlockingUses) {
+ return true;
+}
+
+DeletionKind LLVM::InvariantEndOp::removeBlockingUses(
+ const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+ return DeletionKind::Delete;
+}
+
bool LLVM::DbgDeclareOp::canUsesBeRemoved(
const SmallPtrSetImpl<OpOperand *> &blockingUses,
SmallVectorImpl<OpOperand *> &newBlockingUses) {
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
index 16eb28c629ee23..90e56c1166edfd 100644
--- a/mlir/test/Dialect/LLVMIR/mem2reg.mlir
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -520,6 +520,8 @@ llvm.func @discardable_use_tree() {
%4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
llvm.intr.lifetime.start 2, %3 : !llvm.ptr
llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+ %5 = llvm.intr.invariant.start 2, %3 : !llvm.ptr
+ llvm.intr.invariant.end %5, 2, %3 : !llvm.ptr
llvm.return
}
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 594c3de91815ae..49f34785ebad52 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -577,6 +577,16 @@ llvm.func @lifetime(%p: !llvm.ptr) {
llvm.return
}
+// CHECK-LABEL: @invariant
+// CHECK-SAME: %[[P:.*]]: !llvm.ptr
+llvm.func @invariant(%p: !llvm.ptr) {
+ // CHECK: %[[START:.*]] = llvm.intr.invariant.start 1, %[[P]] : !llvm.ptr
+ %1 = llvm.intr.invariant.start 1, %p : !llvm.ptr
+ // CHECK: llvm.intr.invariant.end %[[START]], 1, %[[P]] : !llvm.ptr
+ llvm.intr.invariant.end %1, 1, %p : !llvm.ptr
+ llvm.return
+}
+
// CHECK-LABEL: @vararg_func
llvm.func @vararg_func(%arg0: i32, ...) {
// CHECK: %[[C:.*]] = llvm.mlir.constant(1 : i32)
diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir
index 09ca94e91d2a12..f56ea53ac029e7 100644
--- a/mlir/test/Dialect/LLVMIR/sroa.mlir
+++ b/mlir/test/Dialect/LLVMIR/sroa.mlir
@@ -170,12 +170,14 @@ llvm.func @direct_promotable_use_is_fine_on_accessor() -> i32 {
// CHECK: %[[ALLOCA:.*]] = llvm.alloca %[[SIZE]] x i32
%1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, f64, i32)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
%2 = llvm.getelementptr %1[0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, f64, i32)>
+ // This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
+ %3 = llvm.intr.invariant.start 2, %2 : !llvm.ptr
// CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
- %3 = llvm.load %2 : !llvm.ptr -> i32
+ %4 = llvm.load %2 : !llvm.ptr -> i32
// This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
- llvm.intr.lifetime.start 2, %2 : !llvm.ptr
+ llvm.intr.invariant.end %3, 2, %2 : !llvm.ptr
// CHECK: llvm.return %[[RES]] : i32
- llvm.return %3 : i32
+ llvm.return %4 : i32
}
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index c8dcde11d93e64..f52ad6b56d141d 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -754,6 +754,15 @@ define void @lifetime(ptr %0) {
ret void
}
+; CHECK-LABEL: llvm.func @invariant
+define void @invariant(ptr %0) {
+ ; CHECK: %[[START:.*]] = llvm.intr.invariant.start 16, %{{.*}} : !llvm.ptr
+ %2 = call ptr @llvm.invariant.start.p0(i64 16, ptr %0)
+ ; CHECK: llvm.intr.invariant.end %[[START]], 32, %{{.*}} : !llvm.ptr
+ call void @llvm.invariant.end.p0(ptr %2, i64 32, ptr %0)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @vector_insert
define void @vector_insert(<vscale x 4 x float> %0, <4 x float> %1) {
; CHECK: llvm.intr.vector.insert %{{.*}}, %{{.*}}[4] : vector<4xf32> into !llvm.vec<? x 4 x f32>
@@ -1096,6 +1105,9 @@ declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32)
declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32)
declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
declare void @llvm.assume(i1)
declare float @llvm.ssa.copy.f32(float returned)
declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index e586c0cd2720e4..1c0aa8d3407ac6 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -941,6 +941,15 @@ llvm.func @lifetime(%p: !llvm.ptr) {
llvm.return
}
+// CHECK-LABEL: @invariant
+llvm.func @invariant(%p: !llvm.ptr) {
+ // CHECK: call ptr @llvm.invariant.start
+ %1 = llvm.intr.invariant.start 16, %p : !llvm.ptr
+ // CHECK: call void @llvm.invariant.end
+ llvm.intr.invariant.end %1, 16, %p : !llvm.ptr
+ llvm.return
+}
+
// CHECK-LABEL: @ssa_copy
llvm.func @ssa_copy(%arg: f32) -> f32 {
// CHECK: call float @llvm.ssa.copy
@@ -1101,6 +1110,9 @@ llvm.func @ssa_copy(%arg: f32) -> f32 {
// CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
// CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
// CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
// CHECK-DAG: declare float @llvm.ssa.copy.f32(float returned)
// CHECK-DAG: declare ptr @llvm.stacksave.p0()
// CHECK-DAG: declare ptr addrspace(1) @llvm.stacksave.p1()
More information about the Mlir-commits
mailing list