[clang] 5f58322 - [HWASan] Build separate LAM runtime on x86_64.
Matt Morehouse via cfe-commits
cfe-commits at lists.llvm.org
Mon May 17 09:19:31 PDT 2021
Author: Matt Morehouse
Date: 2021-05-17T09:19:06-07:00
New Revision: 5f58322368b070b63fe2b2559a54f646cb97e2c4
URL: https://github.com/llvm/llvm-project/commit/5f58322368b070b63fe2b2559a54f646cb97e2c4
DIFF: https://github.com/llvm/llvm-project/commit/5f58322368b070b63fe2b2559a54f646cb97e2c4.diff
LOG: [HWASan] Build separate LAM runtime on x86_64.
Since we have both aliasing mode and Intel LAM on x86_64, we need to
choose the mode at either run time or compile time. This patch
implements the plumbing to build both and choose between them at
compile time.
Reviewed By: vitalybuka, eugenis
Differential Revision: https://reviews.llvm.org/D102286
Added:
Modified:
clang/lib/Driver/ToolChains/CommonArgs.cpp
compiler-rt/lib/hwasan/CMakeLists.txt
compiler-rt/lib/hwasan/hwasan.h
compiler-rt/lib/hwasan/hwasan_allocator.h
compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
compiler-rt/lib/hwasan/hwasan_linux.cpp
Removed:
################################################################################
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index e8fdb08137864..b74a9fe3eb927 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -816,8 +816,12 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
}
if (SanArgs.needsTsanRt() && SanArgs.linkRuntimes())
SharedRuntimes.push_back("tsan");
- if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes())
- SharedRuntimes.push_back("hwasan");
+ if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
+ if (SanArgs.needsHwasanAliasesRt())
+ SharedRuntimes.push_back("hwasan_aliases");
+ else
+ SharedRuntimes.push_back("hwasan");
+ }
}
// The stats_client library is also statically linked into DSOs.
@@ -847,9 +851,15 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
}
if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
- StaticRuntimes.push_back("hwasan");
- if (SanArgs.linkCXXRuntimes())
- StaticRuntimes.push_back("hwasan_cxx");
+ if (SanArgs.needsHwasanAliasesRt()) {
+ StaticRuntimes.push_back("hwasan_aliases");
+ if (SanArgs.linkCXXRuntimes())
+ StaticRuntimes.push_back("hwasan_aliases_cxx");
+ } else {
+ StaticRuntimes.push_back("hwasan");
+ if (SanArgs.linkCXXRuntimes())
+ StaticRuntimes.push_back("hwasan_cxx");
+ }
}
if (SanArgs.needsDfsanRt() && SanArgs.linkRuntimes())
StaticRuntimes.push_back("dfsan");
diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt
index 707cbb44233fe..547b26f0033d6 100644
--- a/compiler-rt/lib/hwasan/CMakeLists.txt
+++ b/compiler-rt/lib/hwasan/CMakeLists.txt
@@ -93,6 +93,24 @@ add_compiler_rt_object_libraries(RTHwasan_dynamic
CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
DEFS ${HWASAN_DEFINITIONS})
+# Compile a
diff erent runtime for x86 aliasing mode.
+set(HWASAN_ALIASES_RTL_CFLAGS ${HWASAN_RTL_CFLAGS})
+list(APPEND HWASAN_ALIASES_RTL_CFLAGS -DHWASAN_ALIASING_MODE)
+set(HWASAN_ALIASES_DYNAMIC_CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+list(APPEND HWASAN_ALIASES_DYNAMIC_CFLAGS -DHWASAN_ALIASING_MODE)
+add_compiler_rt_object_libraries(RTHwasanAliases
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${HWASAN_ALIASES_RTL_CFLAGS}
+ DEFS ${HWASAN_DEFINITIONS})
+add_compiler_rt_object_libraries(RTHwasanAliases_dynamic
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${HWASAN_ALIASES_DYNAMIC_CFLAGS}
+ DEFS ${HWASAN_DEFINITIONS})
+
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
ARCHS ${HWASAN_SUPPORTED_ARCH}
@@ -100,47 +118,61 @@ add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
DEFS ${HWASAN_DEFINITIONS})
-foreach(arch ${HWASAN_SUPPORTED_ARCH})
- add_compiler_rt_runtime(clang_rt.hwasan
+# If use_aliases is TRUE, adds the HWASan runtime built with alias support.
+# Otherwise adds the runtime without alias support.
+function(add_hwasan_runtimes arch use_aliases)
+ set(hwasan_object_lib RTHwasan)
+ set(hwasan_object_dyn_lib RTHwasan_dynamic)
+ set(hwasan_runtime clang_rt.hwasan)
+ set(hwasan_rtl_flags ${HWASAN_RTL_CFLAGS})
+ set(hwasan_dyn_flags ${HWASAN_DYNAMIC_CFLAGS})
+ if(use_aliases)
+ set(hwasan_object_lib RTHwasanAliases)
+ set(hwasan_object_dyn_lib RTHwasanAliases_dynamic)
+ set(hwasan_runtime clang_rt.hwasan_aliases)
+ set(hwasan_rtl_flags ${HWASAN_ALIASES_RTL_CFLAGS})
+ set(hwasan_dyn_flags ${HWASAN_ALIASES_DYNAMIC_CFLAGS})
+ endif()
+ add_compiler_rt_runtime(${hwasan_runtime}
STATIC
ARCHS ${arch}
- OBJECT_LIBS RTHwasan
+ OBJECT_LIBS ${hwasan_object_lib}
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
RTSanitizerCommonCoverage
RTSanitizerCommonSymbolizer
RTUbsan
- CFLAGS ${HWASAN_RTL_CFLAGS}
+ CFLAGS ${hwasan_rtl_flags}
PARENT_TARGET hwasan)
- add_compiler_rt_runtime(clang_rt.hwasan_cxx
+ add_compiler_rt_runtime(${hwasan_runtime}_cxx
STATIC
ARCHS ${arch}
OBJECT_LIBS RTHwasan_cxx
RTUbsan_cxx
- CFLAGS ${HWASAN_RTL_CFLAGS}
+ CFLAGS ${hwasan_rtl_flags}
PARENT_TARGET hwasan)
if (UNIX)
- add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch}
- LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch}
+ add_sanitizer_rt_version_list(${hwasan_runtime}-dynamic-${arch}
+ LIBS ${hwasan_runtime}-${arch} ${hwasan_runtime}_cxx-${arch}
EXTRA hwasan.syms.extra)
set(VERSION_SCRIPT_FLAG
- -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
set_property(SOURCE
${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
APPEND PROPERTY
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
else()
set(VERSION_SCRIPT_FLAG)
endif()
- add_compiler_rt_runtime(clang_rt.hwasan
+ add_compiler_rt_runtime(${hwasan_runtime}
SHARED
ARCHS ${arch}
OBJECT_LIBS
- RTHwasan_dynamic
+ ${hwasan_object_dyn_lib}
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
@@ -154,7 +186,7 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
# add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
# generates an order-only dependency in ninja.
RTHwasan_dynamic_version_script_dummy
- CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
+ CFLAGS ${hwasan_dyn_flags}
LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS}
${VERSION_SCRIPT_FLAG}
LINK_LIBS ${HWASAN_DYNAMIC_LIBS}
@@ -162,14 +194,21 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
PARENT_TARGET hwasan)
if(SANITIZER_USE_SYMBOLS)
- add_sanitizer_rt_symbols(clang_rt.hwasan
+ add_sanitizer_rt_symbols(${hwasan_runtime}
ARCHS ${arch}
EXTRA hwasan.syms.extra)
- add_sanitizer_rt_symbols(clang_rt.hwasan_cxx
+ add_sanitizer_rt_symbols(${hwasan_runtime}_cxx
ARCHS ${arch}
EXTRA hwasan.syms.extra)
- add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols
- clang_rt.hwasan_cxx-${arch}-symbols)
+ add_dependencies(hwasan ${hwasan_runtime}-${arch}-symbols
+ ${hwasan_runtime}_cxx-${arch}-symbols)
+ endif()
+endfunction()
+
+foreach(arch ${HWASAN_SUPPORTED_ARCH})
+ add_hwasan_runtimes(${arch} FALSE)
+ if(${arch} MATCHES "x86_64")
+ add_hwasan_runtimes(${arch} TRUE)
endif()
endforeach()
diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h
index 8515df559f30a..a28a04d3d6edd 100644
--- a/compiler-rt/lib/hwasan/hwasan.h
+++ b/compiler-rt/lib/hwasan/hwasan.h
@@ -36,7 +36,10 @@
typedef u8 tag_t;
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
+# if !defined(__x86_64__)
+# error Aliasing mode is only supported on x86_64
+# endif
// Tags are done in middle bits using userspace aliasing.
constexpr unsigned kAddressTagShift = 39;
constexpr unsigned kTagBits = 3;
@@ -49,12 +52,16 @@ constexpr unsigned kTagBits = 3;
// simpler/faster shadow calculation.
constexpr unsigned kTaggableRegionCheckShift =
__sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#elif defined(__x86_64__)
+// Tags are done in upper bits using Intel LAM.
+constexpr unsigned kAddressTagShift = 57;
+constexpr unsigned kTagBits = 6;
#else
// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
// translation and can be used to store a tag.
constexpr unsigned kAddressTagShift = 56;
constexpr unsigned kTagBits = 8;
-#endif // defined(__x86_64__)
+#endif // defined(HWASAN_ALIASING_MODE)
// Mask for extracting tag bits from the lower 8 bits.
constexpr uptr kTagMask = (1UL << kTagBits) - 1;
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h
index 03bbcff3f0f2e..438a6c5a13c0c 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.h
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.h
@@ -58,7 +58,7 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40; // 1T
struct AP64 {
static const uptr kSpaceBeg = ~0ULL;
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
static const uptr kSpaceSize = 1ULL << kAddressTagShift;
#else
static const uptr kSpaceSize = 0x2000000000ULL;
@@ -110,7 +110,7 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
void GetAllocatorStats(AllocatorStatCounters s);
inline bool InTaggableRegion(uptr addr) {
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
// Aliases are mapped next to shadow so that the upper bits match the shadow
// base.
return (addr >> kTaggableRegionCheckShift) ==
diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
index f53276e330d39..bde22dfa4bc4c 100644
--- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
@@ -119,12 +119,12 @@ namespace __hwasan {
void InitShadowGOT() {}
uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
-#if defined(__x86_64__)
+# if defined(HWASAN_ALIASING_MODE)
constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
constexpr uptr kNumAliases = 1ULL << kTagBits;
return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
RingBufferSize());
-#endif
+# endif
return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
kHighMemEnd);
}
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp
index 8ce0ff7da9569..01c79414a6482 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp
@@ -76,7 +76,7 @@ uptr kHighShadowEnd;
uptr kHighMemStart;
uptr kHighMemEnd;
-uptr kAliasRegionStart; // Always 0 on non-x86.
+uptr kAliasRegionStart; // Always 0 when aliases aren't used.
static void PrintRange(uptr start, uptr end, const char *name) {
Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
@@ -125,33 +125,50 @@ void InitPrctl() {
if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
&local_errno) &&
local_errno == EINVAL) {
-#if SANITIZER_ANDROID || defined(__x86_64__)
+# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
// Some older Android kernels have the tagged pointer ABI on
// unconditionally, and hence don't have the tagged-addr prctl while still
// allow the ABI.
// If targeting Android and the prctl is not around we assume this is the
// case.
return;
-#else
+# else
if (flags()->fail_without_syscall_abi) {
Printf(
"FATAL: "
"HWAddressSanitizer requires a kernel with tagged address ABI.\n");
Die();
}
-#endif
+# endif
}
// Turn on the tagged address ABI.
if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
- !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &&
- flags()->fail_without_syscall_abi) {
- Printf(
- "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
- "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
- "configuration.\n");
- Die();
+ !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
+# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ // Try the new prctl API for Intel LAM. The API is based on a currently
+ // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
+ // subject to change. Patch is here:
+ // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
+ int tag_bits = kTagBits;
+ int tag_shift = kAddressTagShift;
+ if (!internal_iserror(
+ internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
+ reinterpret_cast<unsigned long>(&tag_bits),
+ reinterpret_cast<unsigned long>(&tag_shift), 0))) {
+ CHECK_EQ(tag_bits, kTagBits);
+ CHECK_EQ(tag_shift, kAddressTagShift);
+ return;
+ }
+# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ if (flags()->fail_without_syscall_abi) {
+ Printf(
+ "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+ "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+ "configuration.\n");
+ Die();
+ }
}
#undef PR_SET_TAGGED_ADDR_CTRL
#undef PR_GET_TAGGED_ADDR_CTRL
@@ -181,7 +198,7 @@ bool InitShadow() {
// High memory starts where allocated shadow allows.
kHighMemStart = ShadowToMem(kHighShadowStart);
-#if defined(__x86_64__)
+# if defined(HWASAN_ALIASING_MODE)
constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
kAliasRegionStart =
__hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
@@ -191,7 +208,7 @@ bool InitShadow() {
CHECK_EQ(
(kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
__hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
-#endif
+# endif
// Check the sanity of the defined memory ranges (there might be gaps).
CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
@@ -236,9 +253,11 @@ void InitThreads() {
}
bool MemIsApp(uptr p) {
-#if !defined(__x86_64__) // Memory outside the alias range has non-zero tags.
+// Memory outside the alias range has non-zero tags.
+# if !defined(HWASAN_ALIASING_MODE)
CHECK(GetTagFromPointer(p) == 0);
-#endif
+# endif
+
return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
}
More information about the cfe-commits
mailing list