[llvm] 98eb8ab - Add a type_checked_load_relative to support relative function pointer tables
Arnold Schwaighofer via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 29 08:34:22 PDT 2023
Author: Arnold Schwaighofer
Date: 2023-06-29T08:33:45-07:00
New Revision: 98eb8abff6ad5c2e2f95fd1189b35a24f2800b32
URL: https://github.com/llvm/llvm-project/commit/98eb8abff6ad5c2e2f95fd1189b35a24f2800b32
DIFF: https://github.com/llvm/llvm-project/commit/98eb8abff6ad5c2e2f95fd1189b35a24f2800b32.diff
LOG: Add a type_checked_load_relative to support relative function pointer tables
This adds a type_checked_load_relative intrinsic whose semantics it is to
load a relative function pointer.
A relative function pointer is a pointer to a 32bit value that when
added to its address yields the address of the function.
Differential Revision: https://reviews.llvm.org/D143204
Added:
llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll
llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll
Modified:
llvm/docs/LangRef.rst
llvm/include/llvm/IR/Intrinsics.td
llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
llvm/lib/Analysis/TypeMetadataUtils.cpp
llvm/lib/LTO/LTO.cpp
llvm/lib/Transforms/IPO/GlobalDCE.cpp
llvm/lib/Transforms/IPO/GlobalSplit.cpp
llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
Removed:
################################################################################
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 3e6964ace07a60..25a013254a4218 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -26162,6 +26162,28 @@ element is true, the following rules apply to the first element:
If the function's return value's second element is false, the value of the
first element is undefined.
+.. _type.checked.load.relative:
+
+'``llvm.type.checked.load.relative``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+ declare {ptr, i1} @llvm.type.checked.load.relative(ptr %ptr, i32 %offset, metadata %type) argmemonly nounwind readonly
+
+Overview:
+"""""""""
+
+The ``llvm.type.checked.load.relative`` intrinsic loads a relative pointer to a
+function from a virtual table pointer using metadata. Otherwise, its semantic is
+identical to the ``llvm.type.checked.load`` intrinsic.
+
+A relative pointer is a pointer to an offset to the pointed to value. The
+address of the underlying pointer of the relative pointer is obtained by adding
+the offset to the address of the offset value.
'``llvm.arithmetic.fence``' Intrinsic
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 6acc818e8cb7f2..5ca1ac0751ad7d 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2234,6 +2234,11 @@ def int_type_checked_load : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_i1_ty],
[llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty],
[IntrNoMem, IntrWillReturn]>;
+// Safely loads a relative function pointer from a virtual table pointer using type metadata.
+def int_type_checked_load_relative : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_i1_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty],
+ [IntrNoMem, IntrWillReturn]>;
+
// Test whether a pointer is associated with a type metadata identifier. Used
// for public visibility classes that may later be refined to private
// visibility.
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 3830edc4532552..fd125ac80dd7c0 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -198,6 +198,7 @@ static void addIntrinsicToSummary(
break;
}
+ case Intrinsic::type_checked_load_relative:
case Intrinsic::type_checked_load: {
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp
index 3e7cd712ec973e..bbaee06ed8a552 100644
--- a/llvm/lib/Analysis/TypeMetadataUtils.cpp
+++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp
@@ -99,7 +99,9 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad(
SmallVectorImpl<Instruction *> &Preds, bool &HasNonCallUses,
const CallInst *CI, DominatorTree &DT) {
assert(CI->getCalledFunction()->getIntrinsicID() ==
- Intrinsic::type_checked_load);
+ Intrinsic::type_checked_load ||
+ CI->getCalledFunction()->getIntrinsicID() ==
+ Intrinsic::type_checked_load_relative);
auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
if (!Offset) {
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 6b82c4eaf37be5..65ba3cb3bc1003 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1087,11 +1087,16 @@ Error LTO::checkPartiallySplit() {
Intrinsic::getName(Intrinsic::type_test));
Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
Intrinsic::getName(Intrinsic::type_checked_load));
+ Function *TypeCheckedLoadRelativeFunc =
+ RegularLTO.CombinedModule->getFunction(
+ Intrinsic::getName(Intrinsic::type_checked_load_relative));
// First check if there are type tests / type checked loads in the
// merged regular LTO module IR.
if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
- (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
+ (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
+ (TypeCheckedLoadRelativeFunc &&
+ !TypeCheckedLoadRelativeFunc->use_empty()))
return make_error<StringError>(
"inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
inconvertibleErrorCode());
diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
index 411d3c835600c3..e36d524d7667ab 100644
--- a/llvm/lib/Transforms/IPO/GlobalDCE.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
@@ -187,29 +187,36 @@ void GlobalDCEPass::ScanTypeCheckedLoadIntrinsics(Module &M) {
LLVM_DEBUG(dbgs() << "Scanning type.checked.load intrinsics\n");
Function *TypeCheckedLoadFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
-
- if (!TypeCheckedLoadFunc)
- return;
-
- for (auto *U : TypeCheckedLoadFunc->users()) {
- auto CI = dyn_cast<CallInst>(U);
- if (!CI)
- continue;
-
- auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
- Value *TypeIdValue = CI->getArgOperand(2);
- auto *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
-
- if (Offset) {
- ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue());
- } else {
- // type.checked.load with a non-constant offset, so assume every entry in
- // every matching vtable is used.
- for (const auto &VTableInfo : TypeIdMap[TypeId]) {
- VFESafeVTables.erase(VTableInfo.first);
+ Function *TypeCheckedLoadRelativeFunc =
+ M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
+
+ auto scan = [&](Function *CheckedLoadFunc) {
+ if (!CheckedLoadFunc)
+ return;
+
+ for (auto *U : CheckedLoadFunc->users()) {
+ auto CI = dyn_cast<CallInst>(U);
+ if (!CI)
+ continue;
+
+ auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
+ Value *TypeIdValue = CI->getArgOperand(2);
+ auto *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
+
+ if (Offset) {
+ ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue());
+ } else {
+ // type.checked.load with a non-constant offset, so assume every entry
+ // in every matching vtable is used.
+ for (const auto &VTableInfo : TypeIdMap[TypeId]) {
+ VFESafeVTables.erase(VTableInfo.first);
+ }
}
}
- }
+ };
+
+ scan(TypeCheckedLoadFunc);
+ scan(TypeCheckedLoadRelativeFunc);
}
void GlobalDCEPass::AddVirtualFunctionDependencies(Module &M) {
diff --git a/llvm/lib/Transforms/IPO/GlobalSplit.cpp b/llvm/lib/Transforms/IPO/GlobalSplit.cpp
index c2438dfb618f65..84e9c219f935cd 100644
--- a/llvm/lib/Transforms/IPO/GlobalSplit.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalSplit.cpp
@@ -147,8 +147,12 @@ static bool splitGlobals(Module &M) {
M.getFunction(Intrinsic::getName(Intrinsic::type_test));
Function *TypeCheckedLoadFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
+ Function *TypeCheckedLoadRelativeFunc =
+ M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
if ((!TypeTestFunc || TypeTestFunc->use_empty()) &&
- (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()))
+ (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) &&
+ (!TypeCheckedLoadRelativeFunc ||
+ TypeCheckedLoadRelativeFunc->use_empty()))
return false;
bool Changed = false;
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 4b36c7bce68873..a7adb74801ee59 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -146,6 +146,14 @@ void promoteTypeIds(Module &M, StringRef ModuleId) {
}
}
+ if (Function *TypeCheckedLoadRelativeFunc = M.getFunction(
+ Intrinsic::getName(Intrinsic::type_checked_load_relative))) {
+ for (const Use &U : TypeCheckedLoadRelativeFunc->uses()) {
+ auto CI = cast<CallInst>(U.getUser());
+ ExternalizeTypeId(CI, 2);
+ }
+ }
+
for (GlobalObject &GO : M.global_objects()) {
SmallVector<MDNode *, 1> MDs;
GO.getMetadata(LLVMContext::MD_type, MDs);
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 203846d1936aa3..d332586423651c 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1006,7 +1006,7 @@ bool DevirtModule::tryFindVirtualCallTargets(
return false;
Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(),
- TM.Offset + ByteOffset, M);
+ TM.Offset + ByteOffset, M, TM.Bits->GV);
if (!Ptr)
return false;
@@ -2002,9 +2002,23 @@ void DevirtModule::scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc) {
// This helps avoid unnecessary spills.
IRBuilder<> LoadB(
(LoadedPtrs.size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
- Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
- Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy));
- Value *LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
+
+ Value *LoadedValue = nullptr;
+ if (TypeCheckedLoadFunc->getIntrinsicID() ==
+ Intrinsic::type_checked_load_relative) {
+ Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
+ Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int32Ty));
+ LoadedValue = LoadB.CreateLoad(Int32Ty, GEPPtr);
+ LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy);
+ GEP = LoadB.CreatePtrToInt(GEP, IntPtrTy);
+ LoadedValue = LoadB.CreateAdd(GEP, LoadedValue);
+ LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy);
+ } else {
+ Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
+ Value *GEPPtr =
+ LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy));
+ LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
+ }
for (Instruction *LoadedPtr : LoadedPtrs) {
LoadedPtr->replaceAllUsesWith(LoadedValue);
@@ -2185,6 +2199,8 @@ bool DevirtModule::run() {
M.getFunction(Intrinsic::getName(Intrinsic::type_test));
Function *TypeCheckedLoadFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
+ Function *TypeCheckedLoadRelativeFunc =
+ M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume));
// Normally if there are no users of the devirtualization intrinsics in the
@@ -2193,7 +2209,9 @@ bool DevirtModule::run() {
if (!ExportSummary &&
(!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc ||
AssumeFunc->use_empty()) &&
- (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()))
+ (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) &&
+ (!TypeCheckedLoadRelativeFunc ||
+ TypeCheckedLoadRelativeFunc->use_empty()))
return false;
// Rebuild type metadata into a map for easy lookup.
@@ -2207,6 +2225,9 @@ bool DevirtModule::run() {
if (TypeCheckedLoadFunc)
scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
+ if (TypeCheckedLoadRelativeFunc)
+ scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);
+
if (ImportSummary) {
for (auto &S : CallSlots)
importResolution(S.first, S.second);
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll
new file mode 100644
index 00000000000000..ba8c89d950ffb6
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
+; CHECK: remark: <unknown>:0:0: devirtualized vf
+; CHECK-NOT: devirtualized
+
+; A vtable with "relative pointers", slots don't contain pointers to implementations, but instead have an i32 offset from the vtable itself to the implementation.
+ at vtable = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
+ i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
+ i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc2_dead to i64), i64 ptrtoint (ptr @vtable to i64)) to i32)
+]}, align 8, !type !0, !type !1
+;, !vcall_visibility !{i64 2}
+!0 = !{i64 0, !"vfunc1.type"}
+!1 = !{i64 4, !"vfunc2.type"}
+
+define internal void @vfunc1_live() {
+ ret void
+}
+
+define internal void @vfunc2_dead() {
+ ret void
+}
+
+; CHECK: define void @call
+define void @call(ptr %obj) {
+ %vtable = load ptr, ptr %obj
+ %pair = call {ptr, i1} @llvm.type.checked.load.relative(ptr %vtable, i32 0, metadata !"vfunc1.type")
+ %fptr = extractvalue {ptr, i1} %pair, 0
+ %p = extractvalue {ptr, i1} %pair, 1
+ ; CHECK: br i1 true,
+ br i1 %p, label %cont, label %trap
+
+cont:
+ ; CHECK: call void @vfunc1_live(
+ call void %fptr()
+ ret void
+
+trap:
+ call void @llvm.trap()
+ unreachable
+}
+
+declare {ptr, i1} @llvm.type.checked.load.relative(ptr, i32, metadata)
+declare void @llvm.trap()
diff --git a/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll b/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll
new file mode 100644
index 00000000000000..d67458952ba725
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -passes=wholeprogramdevirt %s | FileCheck %s
+
+; Test that we correctly expand the llvm.type.checked.load.relative intrinsic in
+; cases where we cannot devirtualize.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at vt1 = constant { [2 x i32] } { [2 x i32] [
+ i32 trunc (i64 sub (i64 ptrtoint (ptr @vf1 to i64), i64 ptrtoint (ptr @vt1 to i64)) to i32),
+ i32 trunc (i64 sub (i64 ptrtoint (ptr @vf2 to i64), i64 ptrtoint (ptr @vt1 to i64)) to i32)
+]}, align 8, !type !0, !type !1
+
+!0 = !{i64 0, !"vfunc1.type"}
+!1 = !{i64 4, !"vfunc2.type"}
+
+
+define void @vf1(ptr %this) {
+ ret void
+}
+
+define void @vf2(ptr %this) {
+ ret void
+}
+
+; CHECK: define void @call
+; CHECK: [[TT:%.*]] = call i1 @llvm.type.test(ptr [[VT:%.*]], metadata !"vfunc1.type")
+; CHECK: br i1 [[TT]]
+
+; Relative pointer computation at the address of the i32 value to the i32 value
+; to get to the pointer value.
+
+; CHECK: [[T0:%.*]] = getelementptr i8, ptr [[VT]], i32 0
+; CHECK: [[T1:%.*]] = load i32, ptr [[T0]]
+; CHECK: [[T2:%.*]] = sext i32 [[T1]] to i64
+; CHECK: [[T3:%.*]] = ptrtoint ptr [[T0]] to i64
+; CHECK: [[T4:%.*]] = add i64 [[T3]], [[T2]]
+; CHECK: [[F:%.*]] = inttoptr i64 [[T4]] to ptr
+; CHECK: call void [[F]](ptr
+
+define void @call(ptr %obj) {
+ %vtable = load ptr, ptr %obj
+ %pair = call {ptr, i1} @llvm.type.checked.load.relative(ptr %vtable, i32 0, metadata !"vfunc1.type")
+ %p = extractvalue {ptr, i1} %pair, 1
+ br i1 %p, label %cont, label %trap
+
+cont:
+ %fptr = extractvalue {ptr, i1} %pair, 0
+ call void %fptr(ptr %obj)
+ ret void
+
+trap:
+ call void @llvm.trap()
+ unreachable
+}
+
+declare {ptr, i1} @llvm.type.checked.load.relative(ptr, i32, metadata)
+declare void @llvm.trap()
More information about the llvm-commits
mailing list