[llvm] ff6069b - [JITLink] Add initial native TLS support to ELFNix platform
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 12 23:36:18 PDT 2021
Author: luxufan
Date: 2021-09-13T14:35:49+08:00
New Revision: ff6069b89114325b1904e55d3987701567d522d4
URL: https://github.com/llvm/llvm-project/commit/ff6069b89114325b1904e55d3987701567d522d4
DIFF: https://github.com/llvm/llvm-project/commit/ff6069b89114325b1904e55d3987701567d522d4.diff
LOG: [JITLink] Add initial native TLS support to ELFNix platform
This patch use the same way as the https://reviews.llvm.org/rGfe1fa43f16beac1506a2e73a9f7b3c81179744eb to handle the thread local variable.
It allocates 2 * pointerSize space in GOT to represent the thread key and data address. Instead of using the _tls_get_addr function, I customed a function __orc_rt_elfnix_tls_get_addr to get the address of thread local varible. Currently, this is a wip patch, only one TLS relocation R_X86_64_TLSGD is supported and I need to add the corresponding test cases.
To allocate the TLS descriptor in GOT, I need to get the edge kind information in PerGraphGOTAndPLTStubBuilder, So I add a `Edge::Kind K` argument in some functions in PerGraphGOTAndPLTStubBuilder.h. If it is not suitable, I can think further to solve this problem.
Differential Revision: https://reviews.llvm.org/D109293
Added:
compiler-rt/lib/orc/elfnix_tls.x86-64.S
compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-tls.S
compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-tls.S
llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h
Modified:
compiler-rt/lib/orc/CMakeLists.txt
compiler-rt/lib/orc/elfnix_platform.cpp
llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 2f0e3e182a277..7259f953f3112 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -13,6 +13,7 @@ set(ORC_SOURCES
set(x86_64_SOURCES
# x86-64 specific assembly files will go here.
macho_tlv.x86-64.S
+ elfnix_tls.x86-64.S
)
set(ORC_IMPL_HEADERS
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
index 1a0f22c635239..6d44aa058e6a8 100644
--- a/compiler-rt/lib/orc/elfnix_platform.cpp
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -62,6 +62,10 @@ Error runInitArray(const std::vector<ExecutorAddressRange> &InitArraySections,
return Error::success();
}
+struct TLSInfoEntry {
+ unsigned long Key = 0;
+ unsigned long DataAddress = 0;
+};
class ELFNixPlatformRuntimeState {
private:
@@ -104,12 +108,18 @@ class ELFNixPlatformRuntimeState {
int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
void runAtExits(void *DSOHandle);
+ /// Returns the base address of the section containing ThreadData.
+ Expected<std::pair<const char *, size_t>>
+ getThreadDataSectionFor(const char *ThreadData);
+
private:
PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
PerJITDylibState *getJITDylibStateByName(string_view Path);
PerJITDylibState &
getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
+ Error registerThreadDataSection(span<const char> ThreadDataSection);
+
Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle,
string_view Symbol);
@@ -132,6 +142,9 @@ class ELFNixPlatformRuntimeState {
std::recursive_mutex JDStatesMutex;
std::unordered_map<void *, PerJITDylibState> JDStates;
std::unordered_map<std::string, void *> JDNameToHeader;
+
+ std::mutex ThreadDataSectionsMutex;
+ std::map<const char *, size_t> ThreadDataSections;
};
ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
@@ -156,7 +169,11 @@ Error ELFNixPlatformRuntimeState::registerObjectSections(
if (POSR.EHFrameSection.StartAddress)
__register_frame(POSR.EHFrameSection.StartAddress.toPtr<const char *>());
- // TODO: Register thread data sections.
+ if (POSR.ThreadDataSection.StartAddress) {
+ if (auto Err = registerThreadDataSection(
+ POSR.ThreadDataSection.toSpan<const char>()))
+ return Err;
+ }
return Error::success();
}
@@ -235,6 +252,19 @@ void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
}
}
+Expected<std::pair<const char *, size_t>>
+ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadData);
+ // Check that we have a valid entry conovering this address.
+ if (I == ThreadDataSections.begin())
+ return make_error<StringError>("No thread local data section for key");
+ I = std::prev(I);
+ if (ThreadData >= I->first + I->second)
+ return make_error<StringError>("No thread local data section for key");
+ return *I;
+}
+
ELFNixPlatformRuntimeState::PerJITDylibState *
ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
auto I = JDStates.find(DSOHandle);
@@ -274,6 +304,20 @@ ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
return JDS;
}
+Error ELFNixPlatformRuntimeState::registerThreadDataSection(
+ span<const char> ThreadDataSection) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
+ if (I != ThreadDataSections.begin()) {
+ auto J = std::prev(I);
+ if (J->first + J->second > ThreadDataSection.data())
+ return make_error<StringError>("Overlapping .tdata sections");
+ }
+ ThreadDataSections.insert(
+ I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
+ return Error::success();
+}
+
Expected<ExecutorAddress>
ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
string_view Sym) {
@@ -344,6 +388,42 @@ Error ELFNixPlatformRuntimeState::initializeJITDylib(
return Error::success();
}
+class ELFNixPlatformRuntimeTLVManager {
+public:
+ void *getInstance(const char *ThreadData);
+
+private:
+ std::unordered_map<const char *, char *> Instances;
+ std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
+};
+
+void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
+ auto I = Instances.find(ThreadData);
+ if (I != Instances.end())
+ return I->second;
+ auto TDS =
+ ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
+ if (!TDS) {
+ __orc_rt_log_error(toString(TDS.takeError()).c_str());
+ return nullptr;
+ }
+
+ auto &Allocated = AllocatedSections[TDS->first];
+ if (!Allocated) {
+ Allocated = std::make_unique<char[]>(TDS->second);
+ memcpy(Allocated.get(), TDS->first, TDS->second);
+ }
+ size_t ThreadDataDelta = ThreadData - TDS->first;
+ assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
+
+ char *Instance = Allocated.get() + ThreadDataDelta;
+ Instances[ThreadData] = Instance;
+ return Instance;
+}
+
+void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
+ delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
+}
} // end anonymous namespace
@@ -387,6 +467,39 @@ __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
.release();
}
+//------------------------------------------------------------------------------
+// TLV support
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
+ auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
+ pthread_getspecific(D->Key));
+ if (!TLVMgr)
+ TLVMgr = new ELFNixPlatformRuntimeTLVManager();
+ if (pthread_setspecific(D->Key, TLVMgr)) {
+ __orc_rt_log_error("Call to pthread_setspecific failed");
+ return nullptr;
+ }
+
+ return TLVMgr->getInstance(
+ reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
+ ArgData, ArgSize,
+ []() -> Expected<uint64_t> {
+ pthread_key_t Key;
+ if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
+ __orc_rt_log_error("Call to pthread_key_create failed");
+ return make_error<StringError>(strerror(Err));
+ }
+ return static_cast<uint64_t>(Key);
+ })
+ .release();
+}
+
//------------------------------------------------------------------------------
// cxa_atexit support
//------------------------------------------------------------------------------
diff --git a/compiler-rt/lib/orc/elfnix_tls.x86-64.S b/compiler-rt/lib/orc/elfnix_tls.x86-64.S
new file mode 100644
index 0000000000000..0b0d7b355b8fc
--- /dev/null
+++ b/compiler-rt/lib/orc/elfnix_tls.x86-64.S
@@ -0,0 +1,59 @@
+
+//===-- orc_rt_elfnix_tls_x86-64.s -------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#define REGISTER_SAVE_SPACE_SIZE 512
+
+ .text
+
+ // returns address of TLV in %rax, all other registers preserved
+ .globl ___orc_rt_elfnix_tls_get_addr
+___orc_rt_elfnix_tls_get_addr:
+ pushq %rbp
+ movq %rsp, %rbp
+ subq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ movq %rcx, -16(%rbp)
+ movq %rdx, -24(%rbp)
+ movq %rsi, -32(%rbp)
+ movq %rdi, -40(%rbp)
+ movq %r8, -48(%rbp)
+ movq %r9, -56(%rbp)
+ movq %r10, -64(%rbp)
+ movq %r11, -72(%rbp)
+ movdqa %xmm0, -128(%rbp)
+ movdqa %xmm1, -144(%rbp)
+ movdqa %xmm2, -160(%rbp)
+ movdqa %xmm3, -176(%rbp)
+ movdqa %xmm4, -192(%rbp)
+ movdqa %xmm5, -208(%rbp)
+ movdqa %xmm6, -224(%rbp)
+ movdqa %xmm7, -240(%rbp)
+ call __orc_rt_elfnix_tls_get_addr_impl
+ movq -16(%rbp), %rcx
+ movq -24(%rbp), %rdx
+ movq -32(%rbp), %rsi
+ movq -40(%rbp), %rdi
+ movq -48(%rbp), %r8
+ movq -56(%rbp), %r9
+ movq -64(%rbp), %r10
+ movq -72(%rbp), %r11
+ movdqa -128(%rbp), %xmm0
+ movdqa -144(%rbp), %xmm1
+ movdqa -160(%rbp), %xmm2
+ movdqa -176(%rbp), %xmm3
+ movdqa -192(%rbp), %xmm4
+ movdqa -208(%rbp), %xmm5
+ movdqa -224(%rbp), %xmm6
+ movdqa -240(%rbp), %xmm7
+ addq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ popq %rbp
+ ret
diff --git a/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-tls.S b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-tls.S
new file mode 100644
index 0000000000000..1e9a9383524f0
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/FreeBSD/x86-64/trivial-tls.S
@@ -0,0 +1,81 @@
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+//
+// Test that basic ELF TLS work by adding together TLSs with values
+// 0, 1, and -1, and returning the result (0 for success). This setup
+// tests both zero-initialized (.tbss) and non-zero-initialized
+// (.tdata) sections.
+
+ .text
+ .file "tlstest.cpp"
+ .globl main # -- Begin function main
+ .p2align 4, 0x90
+ .type main, at function
+main: # @main
+# %bb.0: # %entry
+ pushq %rbp
+ movq %rsp, %rbp
+ subq $32, %rsp
+ movl $0, -4(%rbp)
+ movl %edi, -8(%rbp)
+ movq %rsi, -16(%rbp)
+ data16
+ leaq x at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movl (%rax), %eax
+ movl %eax, -24(%rbp) # 4-byte Spill
+ data16
+ leaq y at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movq %rax, %rcx
+ movl -24(%rbp), %eax # 4-byte Reload
+ movl (%rcx), %ecx
+ addl %ecx, %eax
+ movl %eax, -20(%rbp) # 4-byte Spill
+ data16
+ leaq z at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movq %rax, %rcx
+ movl -20(%rbp), %eax # 4-byte Reload
+ movl (%rcx), %ecx
+ addl %ecx, %eax
+ addq $32, %rsp
+ popq %rbp
+ retq
+.Lfunc_end0:
+ .size main, .Lfunc_end0-main
+ # -- End function
+ .type x, at object # @x
+ .section .tbss,"awT", at nobits
+ .globl x
+ .p2align 2
+x:
+ .long 0 # 0x0
+ .size x, 4
+
+ .type y, at object # @y
+ .section .tdata,"awT", at progbits
+ .globl y
+ .p2align 2
+y:
+ .long 1 # 0x1
+ .size y, 4
+
+ .type z, at object # @z
+ .globl z
+ .p2align 2
+z:
+ .long 4294967295 # 0xffffffff
+ .size z, 4
+
+ .section ".note.GNU-stack","", at progbits
+ .addrsig
diff --git a/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-tls.S b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-tls.S
new file mode 100644
index 0000000000000..1e9a9383524f0
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/x86-64/trivial-tls.S
@@ -0,0 +1,81 @@
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+//
+// Test that basic ELF TLS work by adding together TLSs with values
+// 0, 1, and -1, and returning the result (0 for success). This setup
+// tests both zero-initialized (.tbss) and non-zero-initialized
+// (.tdata) sections.
+
+ .text
+ .file "tlstest.cpp"
+ .globl main # -- Begin function main
+ .p2align 4, 0x90
+ .type main, at function
+main: # @main
+# %bb.0: # %entry
+ pushq %rbp
+ movq %rsp, %rbp
+ subq $32, %rsp
+ movl $0, -4(%rbp)
+ movl %edi, -8(%rbp)
+ movq %rsi, -16(%rbp)
+ data16
+ leaq x at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movl (%rax), %eax
+ movl %eax, -24(%rbp) # 4-byte Spill
+ data16
+ leaq y at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movq %rax, %rcx
+ movl -24(%rbp), %eax # 4-byte Reload
+ movl (%rcx), %ecx
+ addl %ecx, %eax
+ movl %eax, -20(%rbp) # 4-byte Spill
+ data16
+ leaq z at TLSGD(%rip), %rdi
+ data16
+ data16
+ rex64
+ callq __tls_get_addr at PLT
+ movq %rax, %rcx
+ movl -20(%rbp), %eax # 4-byte Reload
+ movl (%rcx), %ecx
+ addl %ecx, %eax
+ addq $32, %rsp
+ popq %rbp
+ retq
+.Lfunc_end0:
+ .size main, .Lfunc_end0-main
+ # -- End function
+ .type x, at object # @x
+ .section .tbss,"awT", at nobits
+ .globl x
+ .p2align 2
+x:
+ .long 0 # 0x0
+ .size x, 4
+
+ .type y, at object # @y
+ .section .tdata,"awT", at progbits
+ .globl y
+ .p2align 2
+y:
+ .long 1 # 0x1
+ .size y, 4
+
+ .type z, at object # @z
+ .globl z
+ .p2align 2
+z:
+ .long 4294967295 # 0xffffffff
+ .size z, 4
+
+ .section ".note.GNU-stack","", at progbits
+ .addrsig
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
index 611d58b974494..f5fa9e96c5949 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h
@@ -27,6 +27,7 @@ enum ELFX86RelocationKind : Edge::Kind {
PCRel32GOTLoad,
PCRel32GOTLoadRelaxable,
PCRel32REXGOTLoadRelaxable,
+ PCRel32TLV,
PCRel64GOT,
GOTOFF64,
GOT64,
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
index a11174a7f34ec..1f6a861eb9d3a 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h
@@ -324,6 +324,9 @@ enum EdgeKind_x86_64 : Edge::Kind {
///
PCRel32TLVPLoadREXRelaxable,
+ /// TODO: Explain the generic edge kind
+ RequestTLSDescInGOTAndTransformToDelta32,
+
/// A TLVP entry getter/constructor, transformed to
/// Delta32ToTLVPLoadREXRelaxable.
///
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
index 0ed779c5af756..3673c869647ee 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h
@@ -215,6 +215,8 @@ class ELFNixPlatform : public Platform {
Error registerPerObjectSections(const ELFPerObjectSectionsToRegister &POSR);
+ Expected<uint64_t> createPThreadKey();
+
ExecutionSession &ES;
ObjectLinkingLayer &ObjLinkingLayer;
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
index 337a9d2ee2a68..3a3547663da17 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
@@ -379,7 +379,7 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() {
if (Sym.isDefined() &&
(Sym.getType() == ELF::STT_NOTYPE || Sym.getType() == ELF::STT_FUNC ||
Sym.getType() == ELF::STT_OBJECT ||
- Sym.getType() == ELF::STT_SECTION)) {
+ Sym.getType() == ELF::STT_SECTION || Sym.getType() == ELF::STT_TLS)) {
// FIXME: Handle extended tables.
if (auto *GraphSec = getGraphSection(Sym.st_shndx)) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index e65af56a2f246..f4e1770a61cf0 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -21,6 +21,7 @@
#include "ELFLinkGraphBuilder.h"
#include "JITLinkGeneric.h"
#include "PerGraphGOTAndPLTStubsBuilder.h"
+#include "PerGraphTLSInfoEntryBuilder.h"
#define DEBUG_TYPE "jitlink"
@@ -32,6 +33,56 @@ namespace {
constexpr StringRef ELFGOTSectionName = "$__GOT";
constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
+constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
+
+class PerGraphTLSInfoBuilder_ELF_x86_64
+ : public PerGraphTLSInfoEntryBuilder<PerGraphTLSInfoBuilder_ELF_x86_64> {
+public:
+ static const uint8_t TLSInfoEntryContent[16];
+ using PerGraphTLSInfoEntryBuilder<
+ PerGraphTLSInfoBuilder_ELF_x86_64>::PerGraphTLSInfoEntryBuilder;
+
+ bool isTLSEdgeToFix(Edge &E) {
+ return E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32;
+ }
+
+ Symbol &createTLSInfoEntry(Symbol &Target) {
+ // the TLS Info entry's key value will be written by the fixTLVSectionByName
+ // pass, so create mutable content.
+ auto &TLSInfoEntry = G.createMutableContentBlock(
+ getTLSInfoSection(), G.allocateContent(getTLSInfoEntryContent()), 0, 8,
+ 0);
+ TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
+ return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
+ }
+
+ void fixTLSEdge(Edge &E, Symbol &Target) {
+ if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
+ E.setTarget(Target);
+ E.setKind(x86_64::Delta32);
+ }
+ }
+
+ Section &getTLSInfoSection() const {
+ if (!TLSInfoSection)
+ TLSInfoSection =
+ &G.createSection(ELFTLSInfoSectionName, sys::Memory::MF_READ);
+ return *TLSInfoSection;
+ }
+
+private:
+ ArrayRef<char> getTLSInfoEntryContent() {
+ return {reinterpret_cast<const char *>(TLSInfoEntryContent),
+ sizeof(TLSInfoEntryContent)};
+ }
+
+ mutable Section *TLSInfoSection = nullptr;
+};
+
+const uint8_t PerGraphTLSInfoBuilder_ELF_x86_64::TLSInfoEntryContent[16] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/
+};
class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64
: public PerGraphGOTAndPLTStubsBuilder<
@@ -199,6 +250,8 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64;
case ELF::R_X86_64_PLT32:
return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
+ case ELF::R_X86_64_TLSGD:
+ return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV;
}
return make_error<JITLinkError>("Unsupported x86-64 relocation type " +
formatv("{0:d}: ", Type) +
@@ -315,6 +368,10 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
Addend = 0;
break;
}
+ case PCRel32TLV: {
+ Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
+ break;
+ }
case PCRel32GOTLoadRelaxable: {
Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
Addend = 0;
@@ -481,6 +538,8 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
Config.PrePrunePasses.push_back(markAllSymbolsLive);
// Add an in-place GOT/Stubs pass.
+
+ Config.PostPrunePasses.push_back(PerGraphTLSInfoBuilder_ELF_x86_64::asPass);
Config.PostPrunePasses.push_back(
PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h
new file mode 100644
index 0000000000000..3eea8eb8d0da1
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/PerGraphTLSInfoEntryBuilder.h
@@ -0,0 +1,78 @@
+//===---------------- PerGraphTLSInfoEntryBuilder.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Construct Thread local storage info entry for each graph.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHTLSINFOENTRYBUILDER_H
+#define LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHTLSINFOENTRYBUILDER_H
+
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "jitlink"
+namespace llvm {
+namespace jitlink {
+
+template <typename BuilderImplT> class PerGraphTLSInfoEntryBuilder {
+public:
+ PerGraphTLSInfoEntryBuilder(LinkGraph &G) : G(G) {}
+ static Error asPass(LinkGraph &G) { return BuilderImplT(G).run(); }
+
+ Error run() {
+ LLVM_DEBUG(dbgs() << "Running Per-Graph TLS Info entry builder:\n ");
+
+ std::vector<Block *> Worklist(G.blocks().begin(), G.blocks().end());
+
+ for (auto *B : Worklist)
+ for (auto &E : B->edges()) {
+ if (impl().isTLSEdgeToFix(E)) {
+ LLVM_DEBUG({
+ dbgs() << " Fixing " << G.getEdgeKindName(E.getKind())
+ << " edge at " << formatv("{0:x}", B->getFixupAddress(E))
+ << " (" << formatv("{0:x}", B->getAddress()) << " + "
+ << formatv("{0:x}", E.getOffset()) << ")\n";
+ });
+ impl().fixTLSEdge(E, getTLSInfoEntry(E.getTarget()));
+ }
+ }
+ return Error::success();
+ }
+
+protected:
+ LinkGraph &G;
+
+ Symbol &getTLSInfoEntry(Symbol &Target) {
+ assert(Target.hasName() && "TLS edge cannot point to anonymous target");
+ auto TLSInfoEntryI = TLSInfoEntries.find(Target.getName());
+ if (TLSInfoEntryI == TLSInfoEntries.end()) {
+ auto &TLSInfoEntry = impl().createTLSInfoEntry(Target);
+ LLVM_DEBUG({
+ dbgs() << " Created TLS Info entry for " << Target.getName() << ": "
+ << TLSInfoEntry << "\n";
+ });
+ TLSInfoEntryI =
+ TLSInfoEntries.insert(std::make_pair(Target.getName(), &TLSInfoEntry))
+ .first;
+ }
+ assert(TLSInfoEntryI != TLSInfoEntries.end() &&
+ "Could not get TLSInfo symbol");
+ LLVM_DEBUG({
+ dbgs() << " Using TLS Info entry" << *TLSInfoEntryI->second << "\n";
+ });
+ return *TLSInfoEntryI->second;
+ }
+
+private:
+ DenseMap<StringRef, Symbol *> TLSInfoEntries;
+ BuilderImplT &impl() { return static_cast<BuilderImplT &>(*this); }
+};
+} // namespace jitlink
+} // namespace llvm
+#endif
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index ff8f0e90edf85..815217bbac86f 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -451,7 +451,9 @@ Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
{"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
{"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
{"__orc_rt_elfnix_register_object_sections",
- &orc_rt_elfnix_register_object_sections}};
+ &orc_rt_elfnix_register_object_sections},
+ {"__orc_rt_elfnix_create_pthread_key",
+ &orc_rt_elfnix_create_pthread_key}};
SymbolLookupSet RuntimeSymbols;
std::vector<std::pair<SymbolStringPtr, ExecutorAddress *>> AddrsToRecord;
@@ -546,6 +548,20 @@ Error ELFNixPlatform::registerPerObjectSections(
return ErrResult;
}
+Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
+ if (!orc_rt_elfnix_create_pthread_key)
+ return make_error<StringError>(
+ "Attempting to create pthread key in target, but runtime support has "
+ "not been loaded yet",
+ inconvertibleErrorCode());
+
+ Expected<uint64_t> Result(0);
+ if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
+ orc_rt_elfnix_create_pthread_key.getValue(), Result))
+ return std::move(Err);
+ return Result;
+}
+
void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
jitlink::PassConfiguration &Config) {
@@ -624,8 +640,10 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
// Insert TLV lowering at the start of the PostPrunePasses, since we want
// it to run before GOT/PLT lowering.
- Config.PostPrunePasses.insert(
- Config.PostPrunePasses.begin(),
+
+ // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
+ // pass has done. Because the TLS descriptor need to be allocate in GOT.
+ Config.PostPrunePasses.push_back(
[this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
return fixTLVSectionsAndEdges(G, JD);
});
@@ -754,6 +772,40 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
jitlink::LinkGraph &G, JITDylib &JD) {
// TODO implement TLV support
+ for (auto *Sym : G.external_symbols())
+ if (Sym->getName() == "__tls_get_addr") {
+ Sym->setName("___orc_rt_elfnix_tls_get_addr");
+ }
+
+ auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
+
+ if (TLSInfoEntrySection) {
+ Optional<uint64_t> Key;
+ {
+ std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+ auto I = MP.JITDylibToPThreadKey.find(&JD);
+ if (I != MP.JITDylibToPThreadKey.end())
+ Key = I->second;
+ }
+ if (!Key) {
+ if (auto KeyOrErr = MP.createPThreadKey())
+ Key = *KeyOrErr;
+ else
+ return KeyOrErr.takeError();
+ }
+
+ uint64_t PlatformKeyBits =
+ support::endian::byte_swap(*Key, G.getEndianness());
+
+ for (auto *B : TLSInfoEntrySection->blocks()) {
+ // FIXME: The TLS descriptor byte length may
diff erent with
diff erent
+ // ISA
+ assert(B->getSize() == (G.getPointerSize() * 2) &&
+ "TLS descriptor must be 2 words length");
+ auto TLSInfoEntryContent = B->getMutableContent(G);
+ memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
+ }
+ }
return Error::success();
}
More information about the llvm-commits
mailing list