[llvm] 30b6c51 - [ORC][ORC_RT][AArch64] Implement TLS descriptor in ELFNixPlatform.
Sunho Kim via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 6 04:13:23 PDT 2022
Author: Sunho Kim
Date: 2022-07-06T20:12:22+09:00
New Revision: 30b6c51f515e633d6a798ddb82d4591d597670e9
URL: https://github.com/llvm/llvm-project/commit/30b6c51f515e633d6a798ddb82d4591d597670e9
DIFF: https://github.com/llvm/llvm-project/commit/30b6c51f515e633d6a798ddb82d4591d597670e9.diff
LOG: [ORC][ORC_RT][AArch64] Implement TLS descriptor in ELFNixPlatform.
Implements TLS descriptor relocations in JITLink ELF/AARCH64 backend and support the relevant runtime functions in ELFNixPlatform.
Unlike traditional TLS model, TLS descriptor model requires linker to return the "offset" from thread pointer via relocaiton not the actual pointer to thread local variable. There is no public libc api for adding new allocations to TLS block dynamically which thread pointer points to. So, we support this by taking delta from thread base pointer to the actual thread local variable in our allocated section.
Reviewed By: lhames
Differential Revision: https://reviews.llvm.org/D128601
Added:
compiler-rt/lib/orc/elfnix_tls.aarch64.S
compiler-rt/test/orc/TestCases/Linux/aarch64/trivial-tls.S
Modified:
compiler-rt/lib/orc/CMakeLists.txt
compiler-rt/lib/orc/elfnix_platform.cpp
llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
Removed:
################################################################################
diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index d2786acf3cfe0..b6f2c2d196ac7 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -87,7 +87,10 @@ if (APPLE)
LINK_LIBS ${ORC_LINK_LIBS}
PARENT_TARGET orc)
else() # not Apple
- add_asm_sources(ORC_ASM_SOURCES elfnix_tls.x86-64.S)
+ add_asm_sources(ORC_ASM_SOURCES
+ elfnix_tls.x86-64.S
+ elfnix_tls.aarch64.S
+ )
foreach(arch ${ORC_SUPPORTED_ARCH})
if(NOT CAN_TARGET_${arch})
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
index 6f502b20f8caf..260731ed2732c 100644
--- a/compiler-rt/lib/orc/elfnix_platform.cpp
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -63,11 +63,17 @@ Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
return Error::success();
}
+
struct TLSInfoEntry {
unsigned long Key = 0;
unsigned long DataAddress = 0;
};
+struct TLSDescriptor {
+ void (*Resolver)(void *);
+ TLSInfoEntry *InfoEntry;
+};
+
class ELFNixPlatformRuntimeState {
private:
struct AtExitEntry {
@@ -501,6 +507,13 @@ ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
}
+ORC_RT_INTERFACE ptr
diff _t ___orc_rt_elfnix_tlsdesc_resolver_impl(
+ TLSDescriptor *D, const char *ThreadPointer) {
+ const char *TLVPtr = reinterpret_cast<const char *>(
+ __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
+ return TLVPtr - ThreadPointer;
+}
+
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
diff --git a/compiler-rt/lib/orc/elfnix_tls.aarch64.S b/compiler-rt/lib/orc/elfnix_tls.aarch64.S
new file mode 100644
index 0000000000000..8dcdd535be8ae
--- /dev/null
+++ b/compiler-rt/lib/orc/elfnix_tls.aarch64.S
@@ -0,0 +1,94 @@
+//===-- elfnix_tlv.aarch64.s ---------------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is aarch64-only
+#if defined(__arm64__) || defined(__aarch64__)
+
+#define REGISTER_SAVE_SPACE_SIZE 32 * 24
+
+ .text
+
+ // returns address of TLV in x0, all other registers preserved
+ // TODO: add fast-path for repeat access
+ .globl ___orc_rt_elfnix_tlsdesc_resolver
+___orc_rt_elfnix_tlsdesc_resolver:
+ sub sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ stp x29, x30, [sp, #16 * 1]
+ stp x27, x28, [sp, #16 * 2]
+ stp x25, x26, [sp, #16 * 3]
+ stp x23, x24, [sp, #16 * 4]
+ stp x21, x22, [sp, #16 * 5]
+ stp x19, x20, [sp, #16 * 6]
+ stp x17, x18, [sp, #16 * 7]
+ stp x15, x16, [sp, #16 * 8]
+ stp x13, x14, [sp, #16 * 9]
+ stp x11, x12, [sp, #16 * 10]
+ stp x9, x10, [sp, #16 * 11]
+ stp x7, x8, [sp, #16 * 12]
+ stp x5, x6, [sp, #16 * 13]
+ stp x3, x4, [sp, #16 * 14]
+ stp x1, x2, [sp, #16 * 15]
+ stp q30, q31, [sp, #32 * 8]
+ stp q28, q29, [sp, #32 * 9]
+ stp q26, q27, [sp, #32 * 10]
+ stp q24, q25, [sp, #32 * 11]
+ stp q22, q23, [sp, #32 * 12]
+ stp q20, q21, [sp, #32 * 13]
+ stp q18, q19, [sp, #32 * 14]
+ stp q16, q17, [sp, #32 * 15]
+ stp q14, q15, [sp, #32 * 16]
+ stp q12, q13, [sp, #32 * 17]
+ stp q10, q11, [sp, #32 * 18]
+ stp q8, q9, [sp, #32 * 19]
+ stp q6, q7, [sp, #32 * 20]
+ stp q4, q5, [sp, #32 * 21]
+ stp q2, q3, [sp, #32 * 22]
+ stp q0, q1, [sp, #32 * 23]
+
+ mrs x1, TPIDR_EL0 // get thread pointer
+ bl ___orc_rt_elfnix_tlsdesc_resolver_impl
+
+ ldp q0, q1, [sp, #32 * 23]
+ ldp q2, q3, [sp, #32 * 22]
+ ldp q4, q5, [sp, #32 * 21]
+ ldp q6, q7, [sp, #32 * 20]
+ ldp q8, q9, [sp, #32 * 19]
+ ldp q10, q11, [sp, #32 * 18]
+ ldp q12, q13, [sp, #32 * 17]
+ ldp q14, q15, [sp, #32 * 16]
+ ldp q16, q17, [sp, #32 * 15]
+ ldp q18, q19, [sp, #32 * 14]
+ ldp q20, q21, [sp, #32 * 13]
+ ldp q22, q23, [sp, #32 * 12]
+ ldp q24, q25, [sp, #32 * 11]
+ ldp q26, q27, [sp, #32 * 10]
+ ldp q28, q29, [sp, #32 * 9]
+ ldp q30, q31, [sp, #32 * 8]
+ ldp x1, x2, [sp, #16 * 15]
+ ldp x3, x4, [sp, #16 * 14]
+ ldp x5, x6, [sp, #16 * 13]
+ ldp x7, x8, [sp, #16 * 12]
+ ldp x9, x10, [sp, #16 * 11]
+ ldp x11, x12, [sp, #16 * 10]
+ ldp x13, x14, [sp, #16 * 9]
+ ldp x15, x16, [sp, #16 * 8]
+ ldp x17, x18, [sp, #16 * 7]
+ ldp x19, x20, [sp, #16 * 6]
+ ldp x21, x22, [sp, #16 * 5]
+ ldp x23, x24, [sp, #16 * 4]
+ ldp x25, x26, [sp, #16 * 3]
+ ldp x27, x28, [sp, #16 * 2]
+ ldp x29, x30, [sp, #16 * 1]
+ add sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ ret
+
+#endif // defined(__arm64__) || defined(__aarch64__)
diff --git a/compiler-rt/test/orc/TestCases/Linux/aarch64/trivial-tls.S b/compiler-rt/test/orc/TestCases/Linux/aarch64/trivial-tls.S
new file mode 100644
index 0000000000000..3e6a6b0a9c7fb
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/aarch64/trivial-tls.S
@@ -0,0 +1,66 @@
+// RUN: %clang -c -o %t %s
+// RUN: %llvm_jitlink %t
+//
+// Test that basic ELF TLS work by adding together TLSs with values
+// 0, 1, and -1, and returning the result (0 for success). This setup
+// tests both zero-initialized (.tbss) and non-zero-initialized
+// (.tdata) sections.
+
+ .text
+ .file "tlstest.cpp"
+ .globl main
+ .p2align 2
+ .type main, at function
+main:
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+ adrp x0, :tlsdesc:x
+ ldr x1, [x0, :tlsdesc_lo12:x]
+ add x0, x0, :tlsdesc_lo12:x
+ .tlsdesccall x
+ blr x1
+ mrs x8, TPIDR_EL0
+ ldr w9, [x8, x0]
+ adrp x0, :tlsdesc:y
+ ldr x1, [x0, :tlsdesc_lo12:y]
+ add x0, x0, :tlsdesc_lo12:y
+ .tlsdesccall y
+ blr x1
+ ldr w10, [x8, x0]
+ add w9, w10, w9
+ adrp x0, :tlsdesc:z
+ ldr x1, [x0, :tlsdesc_lo12:z]
+ add x0, x0, :tlsdesc_lo12:z
+ .tlsdesccall z
+ blr x1
+ ldr w8, [x8, x0]
+ add w0, w9, w8
+ ldp x29, x30, [sp], #16
+ ret
+.Lfunc_end0:
+ .size main, .Lfunc_end0-main
+
+ .type x, at object
+ .section .tdata,"awT", at progbits
+ .globl x
+ .p2align 2
+x:
+ .word 4294967295
+ .size x, 4
+
+ .type y, at object
+ .section .tbss,"awT", at nobits
+ .globl y
+ .p2align 2
+y:
+ .word 0
+ .size y, 4
+
+ .type z, at object
+ .section .tdata,"awT", at progbits
+ .globl z
+ .p2align 2
+z:
+ .word 1
+ .size z, 4
+
\ No newline at end of file
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index 53ff6c7a219ee..7262601a7a5fd 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -33,6 +33,8 @@ enum EdgeKind_aarch64 : Edge::Kind {
GOTPageOffset12,
TLVPage21,
TLVPageOffset12,
+ TLSDescPage21,
+ TLSDescPageOffset12,
PointerToGOT,
PairedAddend,
LDRLiteral19,
@@ -223,8 +225,10 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
break;
}
case TLVPage21:
- case GOTPage21:
case TLVPageOffset12:
+ case TLSDescPage21:
+ case TLSDescPageOffset12:
+ case GOTPage21:
case GOTPageOffset12:
case PointerToGOT: {
return make_error<JITLinkError>(
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index 98da3f155c3e7..7d67e5ef343a0 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -63,6 +63,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
ELFPrel64,
ELFAdrGOTPage21,
ELFLd64GOTLo12,
+ ELFTLSDescAdrPage21,
+ ELFTLSDescAddLo12,
+ ELFTLSDescLd64Lo12,
+ ELFTLSDescCall,
};
static Expected<ELFAArch64RelocationKind>
@@ -104,6 +108,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
return ELFAdrGOTPage21;
case ELF::R_AARCH64_LD64_GOT_LO12_NC:
return ELFLd64GOTLo12;
+ case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
+ return ELFTLSDescAdrPage21;
+ case ELF::R_AARCH64_TLSDESC_ADD_LO12:
+ return ELFTLSDescAddLo12;
+ case ELF::R_AARCH64_TLSDESC_LD64_LO12:
+ return ELFTLSDescLd64Lo12;
+ case ELF::R_AARCH64_TLSDESC_CALL:
+ return ELFTLSDescCall;
}
return make_error<JITLinkError>(
@@ -292,6 +304,21 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
Kind = aarch64::GOTPageOffset12;
break;
}
+ case ELFTLSDescAdrPage21: {
+ Kind = aarch64::TLSDescPage21;
+ break;
+ }
+ case ELFTLSDescAddLo12: {
+ Kind = aarch64::TLSDescPageOffset12;
+ break;
+ }
+ case ELFTLSDescLd64Lo12: {
+ Kind = aarch64::TLSDescPageOffset12;
+ break;
+ }
+ case ELFTLSDescCall: {
+ return Error::success();
+ }
};
Edge GE(Kind, Offset, *GraphSymbol, Addend);
@@ -302,6 +329,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
});
BlockToFix.addEdge(std::move(GE));
+
return Error::success();
}
@@ -342,6 +370,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
return "ELFAdrGOTPage21";
case ELFLd64GOTLo12:
return "ELFLd64GOTLo12";
+ case ELFTLSDescAdrPage21:
+ return "ELFTLSDescAdrPage21";
+ case ELFTLSDescAddLo12:
+ return "ELFTLSDescAddLo12";
+ case ELFTLSDescLd64Lo12:
+ return "ELFTLSDescLd64Lo12";
+ case ELFTLSDescCall:
+ return "ELFTLSDescCall";
default:
return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
}
@@ -354,12 +390,133 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
aarch64::getEdgeKindName) {}
};
+// TLS Info Builder.
+class TLSInfoTableManager_ELF_aarch64
+ : public TableManager<TLSInfoTableManager_ELF_aarch64> {
+public:
+ static StringRef getSectionName() { return "$__TLSINFO"; }
+
+ static const uint8_t TLSInfoEntryContent[16];
+
+ bool visitEdge(LinkGraph &G, Block *B, Edge &E) { return false; }
+
+ Symbol &createEntry(LinkGraph &G, Symbol &Target) {
+ // the TLS Info entry's key value will be written by the fixTLVSectionByName
+ // pass, so create mutable content.
+ auto &TLSInfoEntry = G.createMutableContentBlock(
+ getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
+ orc::ExecutorAddr(), 8, 0);
+ TLSInfoEntry.addEdge(aarch64::Pointer64, 8, Target, 0);
+ return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
+ }
+
+private:
+ Section &getTLSInfoSection(LinkGraph &G) {
+ if (!TLSInfoTable)
+ TLSInfoTable = &G.createSection(getSectionName(), MemProt::Read);
+ return *TLSInfoTable;
+ }
+
+ ArrayRef<char> getTLSInfoEntryContent() const {
+ return {reinterpret_cast<const char *>(TLSInfoEntryContent),
+ sizeof(TLSInfoEntryContent)};
+ }
+
+ Section *TLSInfoTable = nullptr;
+};
+
+const uint8_t TLSInfoTableManager_ELF_aarch64::TLSInfoEntryContent[16] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/
+};
+
+// TLS Descriptor Builder.
+class TLSDescTableManager_ELF_aarch64
+ : public TableManager<TLSDescTableManager_ELF_aarch64> {
+public:
+ TLSDescTableManager_ELF_aarch64(
+ TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager)
+ : TLSInfoTableManager(TLSInfoTableManager) {}
+
+ static StringRef getSectionName() { return "$__TLSDESC"; }
+
+ static const uint8_t TLSDescEntryContent[16];
+
+ bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
+ Edge::Kind KindToSet = Edge::Invalid;
+ switch (E.getKind()) {
+ case aarch64::TLSDescPage21: {
+ KindToSet = aarch64::Page21;
+ break;
+ }
+ case aarch64::TLSDescPageOffset12: {
+ KindToSet = aarch64::PageOffset12;
+ break;
+ }
+ default:
+ return false;
+ }
+ assert(KindToSet != Edge::Invalid &&
+ "Fell through switch, but no new kind to set");
+ DEBUG_WITH_TYPE("jitlink", {
+ dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
+ << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
+ << formatv("{0:x}", E.getOffset()) << ")\n";
+ });
+ E.setKind(KindToSet);
+ E.setTarget(getEntryForTarget(G, E.getTarget()));
+ return true;
+ }
+
+ Symbol &createEntry(LinkGraph &G, Symbol &Target) {
+ auto &EntryBlock =
+ G.createContentBlock(getTLSDescSection(G), getTLSDescBlockContent(),
+ orc::ExecutorAddr(), 8, 0);
+ EntryBlock.addEdge(aarch64::Pointer64, 0, getTLSDescResolver(G), 0);
+ EntryBlock.addEdge(aarch64::Pointer64, 8,
+ TLSInfoTableManager.getEntryForTarget(G, Target), 0);
+ return G.addAnonymousSymbol(EntryBlock, 0, 8, false, false);
+ }
+
+private:
+ Section &getTLSDescSection(LinkGraph &G) {
+ if (!GOTSection)
+ GOTSection = &G.createSection(getSectionName(), MemProt::Read);
+ return *GOTSection;
+ }
+
+ Symbol &getTLSDescResolver(LinkGraph &G) {
+ if (!TLSDescResolver)
+ TLSDescResolver =
+ &G.addExternalSymbol("__tlsdesc_resolver", 8, Linkage::Strong);
+ return *TLSDescResolver;
+ }
+
+ ArrayRef<char> getTLSDescBlockContent() {
+ return {reinterpret_cast<const char *>(TLSDescEntryContent),
+ sizeof(TLSDescEntryContent)};
+ }
+
+ Section *GOTSection = nullptr;
+ Symbol *TLSDescResolver = nullptr;
+ TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager;
+};
+
+const uint8_t TLSDescTableManager_ELF_aarch64::TLSDescEntryContent[16] = {
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, /*resolver function pointer*/
+ 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00 /*pointer to tls info*/
+};
+
Error buildTables_ELF_aarch64(LinkGraph &G) {
LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
aarch64::GOTTableManager GOT;
aarch64::PLTTableManager PLT(GOT);
- visitExistingEdges(G, GOT, PLT);
+ TLSInfoTableManager_ELF_aarch64 TLSInfo;
+ TLSDescTableManager_ELF_aarch64 TLSDesc(TLSInfo);
+ visitExistingEdges(G, GOT, PLT, TLSDesc, TLSInfo);
return Error::success();
}
@@ -406,7 +563,7 @@ void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
else
Config.PrePrunePasses.push_back(markAllSymbolsLive);
- // Add an in-place GOT/Stubs build pass.
+ // Add an in-place GOT/TLS/Stubs build pass.
Config.PostPrunePasses.push_back(buildTables_ELF_aarch64);
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index 28a6f9ce90d92..e486bef9fe667 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -48,6 +48,10 @@ const char *getEdgeKindName(Edge::Kind R) {
return "TLVPage21";
case TLVPageOffset12:
return "TLVPageOffset12";
+ case TLSDescPage21:
+ return "TLSDescPage21";
+ case TLSDescPageOffset12:
+ return "TLSDescPageOffset12";
case PointerToGOT:
return "PointerToGOT";
case PairedAddend:
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index e476c549412a0..e7ca636c83e95 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -839,11 +839,13 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
jitlink::LinkGraph &G, JITDylib &JD) {
- // TODO implement TLV support
- for (auto *Sym : G.external_symbols())
+ for (auto *Sym : G.external_symbols()) {
if (Sym->getName() == "__tls_get_addr") {
Sym->setName("___orc_rt_elfnix_tls_get_addr");
+ } else if (Sym->getName() == "__tlsdesc_resolver") {
+ Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
}
+ }
auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
More information about the llvm-commits
mailing list