[libc-commits] [libc] d8e124d - [libc] implement vdso (#91572)
via libc-commits
libc-commits at lists.llvm.org
Wed Sep 11 09:51:14 PDT 2024
Author: Schrodinger ZHU Yifan
Date: 2024-09-11T12:51:11-04:00
New Revision: d8e124dffaaea142d17b9911fc4de91039c8d1b1
URL: https://github.com/llvm/llvm-project/commit/d8e124dffaaea142d17b9911fc4de91039c8d1b1
DIFF: https://github.com/llvm/llvm-project/commit/d8e124dffaaea142d17b9911fc4de91039c8d1b1.diff
LOG: [libc] implement vdso (#91572)
Added:
libc/hdr/link_macros.h
libc/hdr/sys_auxv_macros.h
libc/src/__support/OSUtil/linux/aarch64/vdso.h
libc/src/__support/OSUtil/linux/arm/vdso.h
libc/src/__support/OSUtil/linux/riscv/vdso.h
libc/src/__support/OSUtil/linux/vdso.cpp
libc/src/__support/OSUtil/linux/vdso.h
libc/src/__support/OSUtil/linux/vdso_sym.h
libc/src/__support/OSUtil/linux/x86_64/vdso.h
libc/test/src/__support/OSUtil/linux/vdso_test.cpp
Modified:
libc/hdr/CMakeLists.txt
libc/src/__support/OSUtil/linux/CMakeLists.txt
libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt
libc/src/__support/OSUtil/linux/arm/CMakeLists.txt
libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt
libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
libc/src/sys/auxv/getauxval.h
libc/test/src/__support/OSUtil/linux/CMakeLists.txt
Removed:
################################################################################
diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index a2fad9b473ed7e..e0b65b7c2eb02d 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -143,4 +143,22 @@ add_proxy_header_library(
libc.include.llvm-libc-macros.limits_macros
)
+add_proxy_header_library(
+ link_macros
+ HDRS
+ link_macros.h
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-macros.link_macros
+ libc.include.link
+)
+
+add_proxy_header_library(
+ sys_auxv_macros
+ HDRS
+ sys_auxv_macros.h
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-macros.sys_auxv_macros
+ libc.include.sys_auxv
+)
+
add_subdirectory(types)
diff --git a/libc/hdr/link_macros.h b/libc/hdr/link_macros.h
new file mode 100644
index 00000000000000..8a78a864e6ce4c
--- /dev/null
+++ b/libc/hdr/link_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from link.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_LINK_MACROS_H
+#define LLVM_LIBC_HDR_LINK_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/link-macros.h"
+
+#else // Overlay mode
+
+#include <link.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_LINK_MACROS_H
diff --git a/libc/hdr/sys_auxv_macros.h b/libc/hdr/sys_auxv_macros.h
new file mode 100644
index 00000000000000..c04011baedb860
--- /dev/null
+++ b/libc/hdr/sys_auxv_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from sys/auxv.h ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_SYS_AUXV_MACROS_H
+#define LLVM_LIBC_HDR_SYS_AUXV_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/sys-auxv-macros.h"
+
+#else // Overlay mode
+
+#include <sys/auxv.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_SYS_AUXV_MACROS_H
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index 089cad454d534d..6c7014940407d8 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -23,3 +23,33 @@ add_object_library(
libc.hdr.types.struct_f_owner_ex
libc.hdr.types.off_t
)
+
+add_header_library(
+ vdso_sym
+ HDRS
+ vdso_sym.h
+ DEPENDS
+ libc.src.__support.common
+)
+
+add_object_library(
+ vdso
+ HDRS
+ vdso.h
+ SRCS
+ vdso.cpp
+ DEPENDS
+ .${LIBC_TARGET_ARCHITECTURE}.vdso
+ libc.src.__support.CPP.array
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.string_view
+ libc.src.__support.threads.callonce
+ libc.src.__support.threads.linux.futex_word_type
+ libc.hdr.types.struct_timeval
+ libc.hdr.types.struct_timespec
+ libc.hdr.types.clockid_t
+ libc.hdr.types.time_t
+ libc.hdr.link_macros
+ libc.src.errno.errno
+ libc.src.sys.auxv.getauxval
+)
diff --git a/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt b/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt
index eea9badc46cae6..d9451a1af1df35 100644
--- a/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt
@@ -5,3 +5,13 @@ add_header_library(
DEPENDS
libc.src.__support.common
)
+
+add_header_library(
+ vdso
+ HDRS
+ vdso.h
+ DEPENDS
+ libc.src.__support.common
+ libc.src.__support.CPP.string_view
+ libc.src.__support.OSUtil.linux.vdso_sym
+)
diff --git a/libc/src/__support/OSUtil/linux/aarch64/vdso.h b/libc/src/__support/OSUtil/linux/aarch64/vdso.h
new file mode 100644
index 00000000000000..3c4c6205071da2
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/aarch64/vdso.h
@@ -0,0 +1,37 @@
+//===---------- aarch64 vdso configuration ------------------------* C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/OSUtil/linux/vdso_sym.h"
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+// translate VDSOSym to symbol names
+// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/kernel/vdso/vdso.lds.S
+LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
+ switch (sym) {
+ case VDSOSym::RTSigReturn:
+ return "__kernel_rt_sigreturn";
+ case VDSOSym::GetTimeOfDay:
+ return "__kernel_gettimeofday";
+ case VDSOSym::ClockGetTime:
+ return "__kernel_clock_gettime";
+ case VDSOSym::ClockGetRes:
+ return "__kernel_clock_getres";
+ default:
+ return "";
+ }
+}
+
+// symbol versions
+LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) {
+ return "LINUX_2.6.39";
+}
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H
diff --git a/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt b/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt
index 733366f6d4a2e3..d991f7e0914796 100644
--- a/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt
@@ -5,3 +5,13 @@ add_header_library(
DEPENDS
libc.src.__support.common
)
+
+add_header_library(
+ vdso
+ HDRS
+ vdso.h
+ DEPENDS
+ libc.src.__support.common
+ libc.src.__support.CPP.string_view
+ libc.src.__support.OSUtil.linux.vdso_sym
+)
diff --git a/libc/src/__support/OSUtil/linux/arm/vdso.h b/libc/src/__support/OSUtil/linux/arm/vdso.h
new file mode 100644
index 00000000000000..3de5860359c155
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/arm/vdso.h
@@ -0,0 +1,37 @@
+//===---------- arm vdso configuration ----------------------------* C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/OSUtil/linux/vdso_sym.h"
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+// translate VDSOSym to symbol names
+// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm/vdso/vdso.lds.S
+LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
+ switch (sym) {
+ case VDSOSym::ClockGetTime:
+ return "__vdso_clock_gettime";
+ case VDSOSym::GetTimeOfDay:
+ return "__vdso_gettimeofday";
+ case VDSOSym::ClockGetRes:
+ return "__vdso_clock_getres";
+ case VDSOSym::ClockGetTime64:
+ return "__vdso_clock_gettime64";
+ default:
+ return "";
+ }
+}
+
+// symbol versions
+LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) {
+ return "LINUX_2.6";
+}
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H
diff --git a/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt b/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt
index e271204f519820..eb93dd4af35ce7 100644
--- a/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt
@@ -5,3 +5,13 @@ add_header_library(
DEPENDS
libc.src.__support.common
)
+
+add_header_library(
+ vdso
+ HDRS
+ vdso.h
+ DEPENDS
+ libc.src.__support.common
+ libc.src.__support.CPP.string_view
+ libc.src.__support.OSUtil.linux.vdso_sym
+)
diff --git a/libc/src/__support/OSUtil/linux/riscv/vdso.h b/libc/src/__support/OSUtil/linux/riscv/vdso.h
new file mode 100644
index 00000000000000..24ddb25ea980a5
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/riscv/vdso.h
@@ -0,0 +1,43 @@
+//===---------- RISC-V vdso configuration -------------------------* C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/OSUtil/linux/vdso_sym.h"
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+// translate VDSOSym to symbol names
+// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/kernel/vdso/vdso.lds.S
+LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
+ switch (sym) {
+ case VDSOSym::RTSigReturn:
+ return "__vdso_rt_sigreturn";
+ case VDSOSym::GetTimeOfDay:
+ return "__vdso_gettimeofday";
+ case VDSOSym::ClockGetTime:
+ return "__vdso_clock_gettime";
+ case VDSOSym::ClockGetRes:
+ return "__vdso_clock_getres";
+ case VDSOSym::GetCpu:
+ return "__vdso_getcpu";
+ case VDSOSym::FlushICache:
+ return "__vdso_flush_icache";
+ case VDSOSym::RiscvHwProbe:
+ return "__vdso_riscv_hwprobe";
+ default:
+ return "";
+ }
+}
+
+// symbol versions
+LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) {
+ return "LINUX_4.15";
+}
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H
diff --git a/libc/src/__support/OSUtil/linux/vdso.cpp b/libc/src/__support/OSUtil/linux/vdso.cpp
new file mode 100644
index 00000000000000..cb43764badad1f
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/vdso.cpp
@@ -0,0 +1,237 @@
+//===------------- Linux VDSO Implementation --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "src/__support/OSUtil/linux/vdso.h"
+#include "hdr/link_macros.h"
+#include "hdr/sys_auxv_macros.h"
+#include "src/__support/CPP/array.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/threads/callonce.h"
+#include "src/__support/threads/linux/futex_word.h"
+#include "src/errno/libc_errno.h"
+#include "src/sys/auxv/getauxval.h"
+#include <linux/auxvec.h>
+
+// TODO: This is a temporary workaround to avoid including elf.h
+// Include our own headers for ElfW and friends once we have them.
+namespace LIBC_NAMESPACE_DECL {
+
+namespace vdso {
+
+Symbol::VDSOArray Symbol::global_cache{};
+CallOnceFlag Symbol::once_flag = callonce_impl::NOT_CALLED;
+
+namespace {
+// See https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symverdefs.html
+struct Verdaux {
+ ElfW(Word) vda_name; /* Version or dependency names */
+ ElfW(Word) vda_next; /* Offset in bytes to next verdaux
+ entry */
+};
+struct Verdef {
+ ElfW(Half) vd_version; /* Version revision */
+ ElfW(Half) vd_flags; /* Version information */
+ ElfW(Half) vd_ndx; /* Version Index */
+ ElfW(Half) vd_cnt; /* Number of associated aux entries */
+ ElfW(Word) vd_hash; /* Version name hash value */
+ ElfW(Word) vd_aux; /* Offset in bytes to verdaux array */
+ ElfW(Word) vd_next; /* Offset in bytes to next verdef entry */
+ Verdef *next() const {
+ if (vd_next == 0)
+ return nullptr;
+ return reinterpret_cast<Verdef *>(reinterpret_cast<uintptr_t>(this) +
+ vd_next);
+ }
+ Verdaux *aux() const {
+ return reinterpret_cast<Verdaux *>(reinterpret_cast<uintptr_t>(this) +
+ vd_aux);
+ }
+};
+
+// version search procedure specified by
+// https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symversion.html#SYMVERTBL
+cpp::string_view find_version(Verdef *verdef, ElfW(Half) * versym,
+ const char *strtab, size_t idx) {
+ constexpr ElfW(Half) VER_FLG_BASE = 0x1;
+ if (!versym)
+ return "";
+ ElfW(Half) identifier = versym[idx] & 0x7FFF;
+ // iterate through all version definitions
+ for (Verdef *def = verdef; def != nullptr; def = def->next()) {
+ // skip if this is a file-level version
+ if (def->vd_flags & VER_FLG_BASE)
+ continue;
+ // check if the version identifier matches. Highest bit is used to determine
+ // whether the symbol is local. Only lower 15 bits are used for version
+ // identifier.
+ if ((def->vd_ndx & 0x7FFF) == identifier) {
+ Verdaux *aux = def->aux();
+ return strtab + aux->vda_name;
+ }
+ }
+ return "";
+}
+
+size_t shdr_get_symbol_count(ElfW(Shdr) * vdso_shdr, size_t e_shnum) {
+ if (!vdso_shdr)
+ return 0;
+ // iterate all sections until we locate the dynamic symbol section
+ for (size_t i = 0; i < e_shnum; ++i) {
+ // dynamic symbol section is a table section
+ // therefore, the number of entries can be computed as the ratio
+ // of the section size to the size of a single entry
+ if (vdso_shdr[i].sh_type == SHT_DYNSYM)
+ return vdso_shdr[i].sh_size / vdso_shdr[i].sh_entsize;
+ }
+ return 0;
+}
+
+struct VDSOSymbolTable {
+ const char *strtab;
+ ElfW(Sym) * symtab;
+ // The following can be nullptr if the vDSO does not have versioning
+ ElfW(Half) * versym;
+ Verdef *verdef;
+
+ void populate_symbol_cache(Symbol::VDSOArray &symbol_table,
+ size_t symbol_count, ElfW(Addr) vdso_addr) {
+ for (size_t i = 0, e = symbol_table.size(); i < e; ++i) {
+ Symbol sym = i;
+ cpp::string_view name = sym.name();
+ cpp::string_view version = sym.version();
+ if (name.empty())
+ continue;
+
+ for (size_t j = 0; j < symbol_count; ++j) {
+ if (name == strtab + symtab[j].st_name) {
+ // we find a symbol with desired name
+ // now we need to check if it has the right version
+ if (versym && verdef &&
+ version != find_version(verdef, versym, strtab, j))
+ continue;
+
+ // put the symbol address into the symbol table
+ symbol_table[i] =
+ reinterpret_cast<void *>(vdso_addr + symtab[j].st_value);
+ }
+ }
+ }
+ }
+};
+
+struct PhdrInfo {
+ ElfW(Addr) vdso_addr;
+ ElfW(Dyn) * vdso_dyn;
+ static cpp::optional<PhdrInfo> from(ElfW(Phdr) * vdso_phdr, size_t e_phnum,
+ uintptr_t vdso_ehdr_addr) {
+ constexpr ElfW(Addr) INVALID_ADDR = static_cast<ElfW(Addr)>(-1);
+ ElfW(Addr) vdso_addr = INVALID_ADDR;
+ ElfW(Dyn) *vdso_dyn = nullptr;
+ if (!vdso_phdr)
+ return cpp::nullopt;
+ // iterate through all the program headers until we get the desired pieces
+ for (size_t i = 0; i < e_phnum; ++i) {
+ if (vdso_phdr[i].p_type == PT_DYNAMIC)
+ vdso_dyn = reinterpret_cast<ElfW(Dyn) *>(vdso_ehdr_addr +
+ vdso_phdr[i].p_offset);
+
+ if (vdso_phdr[i].p_type == PT_LOAD)
+ vdso_addr =
+ vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr;
+
+ if (vdso_addr && vdso_dyn)
+ return PhdrInfo{vdso_addr, vdso_dyn};
+ }
+
+ return cpp::nullopt;
+ }
+
+ cpp::optional<VDSOSymbolTable> populate_symbol_table() {
+ const char *strtab = nullptr;
+ ElfW(Sym) *symtab = nullptr;
+ ElfW(Half) *versym = nullptr;
+ Verdef *verdef = nullptr;
+ for (ElfW(Dyn) *d = vdso_dyn; d->d_tag != DT_NULL; ++d) {
+ switch (d->d_tag) {
+ case DT_STRTAB:
+ strtab = reinterpret_cast<const char *>(vdso_addr + d->d_un.d_ptr);
+ break;
+ case DT_SYMTAB:
+ symtab = reinterpret_cast<ElfW(Sym) *>(vdso_addr + d->d_un.d_ptr);
+ break;
+ case DT_VERSYM:
+ versym = reinterpret_cast<uint16_t *>(vdso_addr + d->d_un.d_ptr);
+ break;
+ case DT_VERDEF:
+ verdef = reinterpret_cast<Verdef *>(vdso_addr + d->d_un.d_ptr);
+ break;
+ }
+ if (strtab && symtab && versym && verdef)
+ break;
+ }
+ if (strtab == nullptr || symtab == nullptr)
+ return cpp::nullopt;
+
+ return VDSOSymbolTable{strtab, symtab, versym, verdef};
+ }
+};
+} // namespace
+
+void Symbol::initialize_vdso_global_cache() {
+ // first clear the symbol table
+ for (auto &i : global_cache)
+ i = nullptr;
+
+ // get the address of the VDSO, protect errno since getauxval may change
+ // it
+ int errno_backup = libc_errno;
+ uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
+ // Get the memory address of the vDSO ELF header.
+ auto vdso_ehdr = reinterpret_cast<ElfW(Ehdr) *>(vdso_ehdr_addr);
+ // leave the table unpopulated if we don't have vDSO
+ if (vdso_ehdr == nullptr) {
+ libc_errno = errno_backup;
+ return;
+ }
+
+ // locate the section header inside the elf using the section header
+ // offset
+ auto vdso_shdr =
+ reinterpret_cast<ElfW(Shdr) *>(vdso_ehdr_addr + vdso_ehdr->e_shoff);
+ size_t symbol_count = shdr_get_symbol_count(vdso_shdr, vdso_ehdr->e_shnum);
+
+ // early return if no symbol is found
+ if (symbol_count == 0)
+ return;
+
+ // We need to find both the loadable segment and the dynamic linking of
+ // the vDSO. compute vdso_phdr as the program header using the program
+ // header offset
+ ElfW(Phdr) *vdso_phdr =
+ reinterpret_cast<ElfW(Phdr) *>(vdso_ehdr_addr + vdso_ehdr->e_phoff);
+ cpp::optional<PhdrInfo> phdr_info =
+ PhdrInfo::from(vdso_phdr, vdso_ehdr->e_phnum, vdso_ehdr_addr);
+ // early return if either the dynamic linking or the loadable segment is
+ // not found
+ if (!phdr_info.has_value())
+ return;
+
+ // now, locate several more tables inside the dynmaic linking section
+ cpp::optional<VDSOSymbolTable> vdso_symbol_table =
+ phdr_info->populate_symbol_table();
+
+ // early return if we can't find any required fields of the symbol table
+ if (!vdso_symbol_table.has_value())
+ return;
+
+ // finally, populate the global symbol table cache
+ vdso_symbol_table->populate_symbol_cache(global_cache, symbol_count,
+ phdr_info->vdso_addr);
+}
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/linux/vdso.h b/libc/src/__support/OSUtil/linux/vdso.h
new file mode 100644
index 00000000000000..a5108b3a1fb5d3
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/vdso.h
@@ -0,0 +1,81 @@
+//===------------- Linux VDSO Header ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H
+#include "src/__support/CPP/array.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/properties/architectures.h"
+#include "src/__support/threads/callonce.h"
+
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+#include "x86_64/vdso.h"
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#include "aarch64/vdso.h"
+#elif defined(LIBC_TARGET_ARCH_IS_ARM)
+#include "arm/vdso.h"
+#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
+#include "riscv/vdso.h"
+#else
+#error "unknown arch"
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+
+class Symbol {
+ VDSOSym sym;
+
+public:
+ LIBC_INLINE_VAR static constexpr size_t COUNT =
+ static_cast<size_t>(VDSOSym::VDSOSymCount);
+ LIBC_INLINE constexpr explicit Symbol(VDSOSym sym) : sym(sym) {}
+ LIBC_INLINE constexpr Symbol(size_t idx) : sym(static_cast<VDSOSym>(idx)) {}
+ LIBC_INLINE constexpr cpp::string_view name() const {
+ return symbol_name(sym);
+ }
+ LIBC_INLINE constexpr cpp::string_view version() const {
+ return symbol_version(sym);
+ }
+ LIBC_INLINE constexpr operator size_t() const {
+ return static_cast<size_t>(sym);
+ }
+ LIBC_INLINE constexpr bool is_valid() const {
+ return *this < Symbol::global_cache.size();
+ }
+ using VDSOArray = cpp::array<void *, Symbol::COUNT>;
+
+private:
+ static CallOnceFlag once_flag;
+ static VDSOArray global_cache;
+ static void initialize_vdso_global_cache();
+
+ LIBC_INLINE void *get() const {
+ if (name().empty() || !is_valid())
+ return nullptr;
+
+ callonce(&once_flag, Symbol::initialize_vdso_global_cache);
+ return (global_cache[*this]);
+ }
+ template <VDSOSym sym> friend struct TypedSymbol;
+};
+
+template <VDSOSym sym> struct TypedSymbol {
+ LIBC_INLINE constexpr operator VDSOSymType<sym>() const {
+ return cpp::bit_cast<VDSOSymType<sym>>(Symbol{sym}.get());
+ }
+ template <typename... Args>
+ LIBC_INLINE auto operator()(Args &&...args) const {
+ return this->operator VDSOSymType<sym>()(cpp::forward<Args>(args)...);
+ }
+};
+
+} // namespace vdso
+
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H
diff --git a/libc/src/__support/OSUtil/linux/vdso_sym.h b/libc/src/__support/OSUtil/linux/vdso_sym.h
new file mode 100644
index 00000000000000..eb5f204a82f304
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/vdso_sym.h
@@ -0,0 +1,70 @@
+//===------------- Linux VDSO Symbols ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "hdr/types/clock_t.h"
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
+#include "hdr/types/struct_timeval.h"
+#include "hdr/types/time_t.h"
+#include "src/__support/common.h"
+#include <stddef.h> // For size_t.
+
+// NOLINTBEGIN(llvmlibc-implementation-in-namespace)
+// TODO: some of the following can be defined via proxy headers.
+struct __kernel_timespec;
+struct timezone;
+struct riscv_hwprobe;
+struct getcpu_cache;
+struct cpu_set_t;
+// NOLINTEND(llvmlibc-implementation-in-namespace)
+
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+
+enum class VDSOSym {
+ ClockGetTime,
+ ClockGetTime64,
+ GetTimeOfDay,
+ GetCpu,
+ Time,
+ ClockGetRes,
+ RTSigReturn,
+ FlushICache,
+ RiscvHwProbe,
+ VDSOSymCount
+};
+
+template <VDSOSym sym> LIBC_INLINE constexpr auto dispatcher() {
+ if constexpr (sym == VDSOSym::ClockGetTime)
+ return static_cast<int (*)(clockid_t, timespec *)>(nullptr);
+ else if constexpr (sym == VDSOSym::ClockGetTime64)
+ return static_cast<int (*)(clockid_t, __kernel_timespec *)>(nullptr);
+ else if constexpr (sym == VDSOSym::GetTimeOfDay)
+ return static_cast<int (*)(timeval *__restrict, timezone *__restrict)>(
+ nullptr);
+ else if constexpr (sym == VDSOSym::GetCpu)
+ return static_cast<int (*)(unsigned *, unsigned *, getcpu_cache *)>(
+ nullptr);
+ else if constexpr (sym == VDSOSym::Time)
+ return static_cast<time_t (*)(time_t *)>(nullptr);
+ else if constexpr (sym == VDSOSym::ClockGetRes)
+ return static_cast<int (*)(clockid_t, timespec *)>(nullptr);
+ else if constexpr (sym == VDSOSym::RTSigReturn)
+ return static_cast<void (*)(void)>(nullptr);
+ else if constexpr (sym == VDSOSym::FlushICache)
+ return static_cast<void (*)(void *, void *, unsigned int)>(nullptr);
+ else if constexpr (sym == VDSOSym::RiscvHwProbe)
+ return static_cast<int (*)(riscv_hwprobe *, size_t, size_t, cpu_set_t *,
+ unsigned)>(nullptr);
+ else
+ return static_cast<void *>(nullptr);
+}
+
+template <VDSOSym sym> using VDSOSymType = decltype(dispatcher<sym>());
+
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt b/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
index a7f2d74e6353e0..1324491f37b76b 100644
--- a/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
@@ -5,3 +5,13 @@ add_header_library(
DEPENDS
libc.src.__support.common
)
+
+add_header_library(
+ vdso
+ HDRS
+ vdso.h
+ DEPENDS
+ libc.src.__support.common
+ libc.src.__support.CPP.string_view
+ libc.src.__support.OSUtil.linux.vdso_sym
+)
diff --git a/libc/src/__support/OSUtil/linux/x86_64/vdso.h b/libc/src/__support/OSUtil/linux/x86_64/vdso.h
new file mode 100644
index 00000000000000..abe7c33e07cfab
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/x86_64/vdso.h
@@ -0,0 +1,43 @@
+//===---------- x86/x86_64 vdso configuration ---------------------* C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H
+#include "src/__support/CPP/string_view.h"
+#include "src/__support/OSUtil/linux/vdso_sym.h"
+namespace LIBC_NAMESPACE_DECL {
+namespace vdso {
+// translate VDSOSym to symbol names
+// On x86, there are symbols defined without the __vdso_ prefix, however,
+// it is suggested that one should use the __vdso_ prefix.
+// Additionally, there is also an __vdso_sgx_enter_enclave, it is for the SGX
+// support, we do not include it here for now.
+// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/entry/vdso/vdso.lds.S
+LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
+ switch (sym) {
+ case VDSOSym::ClockGetTime:
+ return "__vdso_clock_gettime";
+ case VDSOSym::GetTimeOfDay:
+ return "__vdso_gettimeofday";
+ case VDSOSym::GetCpu:
+ return "__vdso_getcpu";
+ case VDSOSym::Time:
+ return "__vdso_time";
+ case VDSOSym::ClockGetRes:
+ return "__vdso_clock_getres";
+ default:
+ return "";
+ }
+}
+
+// symbol versions
+LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) {
+ return "LINUX_2.6";
+}
+} // namespace vdso
+} // namespace LIBC_NAMESPACE_DECL
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H
diff --git a/libc/src/sys/auxv/getauxval.h b/libc/src/sys/auxv/getauxval.h
index 3e6971340bbef1..d9da45ff083981 100644
--- a/libc/src/sys/auxv/getauxval.h
+++ b/libc/src/sys/auxv/getauxval.h
@@ -9,8 +9,8 @@
#ifndef LLVM_LIBC_SRC_SYS_AUXV_GETAUXVAL_H
#define LLVM_LIBC_SRC_SYS_AUXV_GETAUXVAL_H
+#include "hdr/sys_auxv_macros.h"
#include "src/__support/macros/config.h"
-#include <sys/auxv.h>
namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/test/src/__support/OSUtil/linux/CMakeLists.txt b/libc/test/src/__support/OSUtil/linux/CMakeLists.txt
index bfb072c03e9712..ff82616cc4a701 100644
--- a/libc/test/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/test/src/__support/OSUtil/linux/CMakeLists.txt
@@ -1,3 +1,21 @@
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
add_subdirectory(${LIBC_TARGET_ARCHITECTURE})
endif()
+
+add_libc_test(
+ vdso_test
+ SUITE libc-osutil-tests
+ SRCS vdso_test.cpp
+ DEPENDS
+ libc.src.__support.OSUtil.linux.vdso
+ libc.src.__support.OSUtil.osutil
+ libc.hdr.types.struct_sigaction
+ libc.hdr.types.struct_timeval
+ libc.hdr.types.struct_timespec
+ libc.hdr.types.clockid_t
+ libc.hdr.types.time_t
+ libc.hdr.time_macros
+ libc.hdr.signal_macros
+ libc.src.signal.sigaction
+ libc.src.signal.raise
+)
diff --git a/libc/test/src/__support/OSUtil/linux/vdso_test.cpp b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp
new file mode 100644
index 00000000000000..2363db69c02f97
--- /dev/null
+++ b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp
@@ -0,0 +1,162 @@
+//===-- Unittests for VDSO ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/signal_macros.h"
+#include "hdr/time_macros.h"
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_sigaction.h"
+#include "hdr/types/struct_timespec.h"
+#include "hdr/types/struct_timeval.h"
+#include "hdr/types/time_t.h"
+#include "src/__support/OSUtil/linux/vdso.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/macros/properties/architectures.h"
+#include "src/signal/raise.h"
+#include "src/signal/sigaction.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/LibcTest.h"
+#include "test/UnitTest/Test.h"
+#include <linux/time_types.h>
+#include <sys/syscall.h>
+
+struct riscv_hwprobe {
+ int64_t key;
+ uint64_t value;
+};
+
+namespace LIBC_NAMESPACE_DECL {
+// For x86_64, we explicitly test some traditional vdso symbols are indeed
+// available.
+
+TEST(LlvmLibcOSUtilVDSOTest, GetTimeOfDay) {
+ vdso::TypedSymbol<vdso::VDSOSym::GetTimeOfDay> symbol;
+ if (!symbol)
+ return;
+ timeval tv;
+ EXPECT_EQ(symbol(&tv, nullptr), 0);
+ // hopefully people are not building time machines using our libc.
+ EXPECT_GT(tv.tv_sec, static_cast<decltype(tv.tv_sec)>(0));
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, Time) {
+ vdso::TypedSymbol<vdso::VDSOSym::Time> symbol;
+ if (!symbol)
+ return;
+ time_t a, b;
+ EXPECT_GT(symbol(&a), static_cast<time_t>(0));
+ EXPECT_GT(symbol(&b), static_cast<time_t>(0));
+ EXPECT_GE(b, a);
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, ClockGetTime) {
+ vdso::TypedSymbol<vdso::VDSOSym::ClockGetTime> symbol;
+ if (!symbol)
+ return;
+ timespec a, b;
+ EXPECT_EQ(symbol(CLOCK_MONOTONIC, &a), 0);
+ EXPECT_EQ(symbol(CLOCK_MONOTONIC, &b), 0);
+ if (a.tv_sec == b.tv_sec) {
+ EXPECT_LT(a.tv_nsec, b.tv_nsec);
+ } else {
+ EXPECT_LT(a.tv_sec, b.tv_sec);
+ }
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, ClockGetTime64) {
+ vdso::TypedSymbol<vdso::VDSOSym::ClockGetTime64> symbol;
+ if (!symbol)
+ return;
+ // See kernel API at
+ // https://elixir.bootlin.com/linux/latest/source/tools/testing/selftests/vDSO/vdso_test_correctness.c#L155
+ __kernel_timespec a, b;
+ EXPECT_EQ(symbol(CLOCK_MONOTONIC, &a), 0);
+ EXPECT_EQ(symbol(CLOCK_MONOTONIC, &b), 0);
+ if (a.tv_sec == b.tv_sec) {
+ EXPECT_LT(a.tv_nsec, b.tv_nsec);
+ } else {
+ EXPECT_LT(a.tv_sec, b.tv_sec);
+ }
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, ClockGetRes) {
+ vdso::TypedSymbol<vdso::VDSOSym::ClockGetRes> symbol;
+ if (!symbol)
+ return;
+ timespec res{};
+ EXPECT_EQ(symbol(CLOCK_MONOTONIC, &res), 0);
+ EXPECT_TRUE(res.tv_sec > 0 || res.tv_nsec > 0);
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, GetCpu) {
+ // The kernel system call has a third argument, which should be passed as
+ // nullptr.
+ vdso::TypedSymbol<vdso::VDSOSym::GetCpu> symbol;
+ if (!symbol)
+ return;
+ unsigned cpu = static_cast<unsigned>(-1), node = static_cast<unsigned>(-1);
+ EXPECT_EQ(symbol(&cpu, &node, nullptr), 0);
+ EXPECT_GE(cpu, 0u);
+ EXPECT_GE(node, 0u);
+}
+
+static bool flag = false;
+static void sigprof_handler [[gnu::used]] (int) { flag = true; }
+
+TEST(LlvmLibcOSUtilVDSOTest, RtSigReturn) {
+ using namespace testing::ErrnoSetterMatcher;
+ // must use struct since there is a function of the same name in the same
+ // scope.
+ struct sigaction sa {};
+ struct sigaction old_sa {};
+ sa.sa_handler = sigprof_handler;
+ sa.sa_flags = SA_RESTORER;
+ vdso::TypedSymbol<vdso::VDSOSym::RTSigReturn> symbol;
+ if (!symbol)
+ return;
+ sa.sa_restorer = symbol;
+ ASSERT_THAT(LIBC_NAMESPACE::sigaction(SIGPROF, &sa, &old_sa), Succeeds());
+ raise(SIGPROF);
+ ASSERT_TRUE(flag);
+ flag = false;
+ ASSERT_THAT(LIBC_NAMESPACE::sigaction(SIGPROF, &old_sa, nullptr), Succeeds());
+}
+
+TEST(LlvmLibcOSUtilVDSOTest, FlushICache) {
+ vdso::TypedSymbol<vdso::VDSOSym::FlushICache> symbol;
+ if (!symbol)
+ return;
+ char buf[512];
+ // we just check that the flush will not panic the program.
+ // the flags part only take 0/1 as up to kernel 6.10, which is used to
+ // indicate whether the flush is local to the core or global.
+ symbol(buf, buf + sizeof(buf), 0);
+ symbol(buf, buf + sizeof(buf), 1);
+}
+
+// https://docs.kernel.org/6.5/riscv/hwprobe.html
+TEST(LlvmLibcOSUtilVDSOTest, RiscvHwProbe) {
+ using namespace testing::ErrnoSetterMatcher;
+ vdso::TypedSymbol<vdso::VDSOSym::RiscvHwProbe> symbol;
+ if (!symbol)
+ return;
+ // If a key is unknown to the kernel, its key field will be cleared to -1, and
+ // its value set to 0. We expect probes.value are all 0.
+ // Usermode can supply NULL for cpus and 0 for cpu_count as a shortcut for all
+ // online CPUs
+ riscv_hwprobe probes[2] = {{-1, 1}, {-1, 1}};
+ ASSERT_THAT(symbol(/*pairs=*/probes, /*count=*/2, /*cpusetsize=*/0,
+ /*cpuset=*/nullptr,
+ /*flags=*/0),
+ Succeeds());
+ for (auto &probe : probes) {
+ EXPECT_EQ(probe.key, static_cast<decltype(probe.key)>(-1));
+ EXPECT_EQ(probe.value, static_cast<decltype(probe.value)>(0));
+ }
+}
+
+} // namespace LIBC_NAMESPACE_DECL
More information about the libc-commits
mailing list